{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Replication for results in Davidson et al. 2017. \"Automated Hate Speech Detection and the Problem of Offensive Language\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import pickle\n",
    "import sys\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "import nltk\n",
    "from nltk.stem.porter import *\n",
    "import string\n",
    "import re\n",
    "from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer as VS\n",
    "from textstat.textstat import *\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import LinearSVC\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"../data/labeled_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>count</th>\n",
       "      <th>hate_speech</th>\n",
       "      <th>offensive_language</th>\n",
       "      <th>neither</th>\n",
       "      <th>class</th>\n",
       "      <th>tweet</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>!!! RT @mayasolovely: As a woman you shouldn't...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!! RT @mleew17: boy dats cold...tyga dwn ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!!!! RT @C_G_Anderson: @viva_based she lo...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>5</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!!!!!!!!!!!!!\"@T_Madison_x: The shit just...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>6</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!!!\"@__BrighterDays: I can not just sit up ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>7</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>!!!!&amp;#8220;@selfiequeenbri: cause I'm tired of...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" &amp;amp; you might not get ya bitch back &amp;amp; ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>9</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" @rhythmixx_ :hobbies include: fighting Maria...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" Keeks is a bitch she curves everyone \" lol I...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>11</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" Murda Gang bitch its Gang Land \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>12</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>\" So hoes that smoke are losers ? \" yea ... go...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>13</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bad bitches is the only thing that i like \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>14</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitch get up off me \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>15</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitch nigga miss me with it \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>16</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitch plz whatever \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>17</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitch who do you love \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>18</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" bitches get cut off everyday B \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>19</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" black bottle &amp;amp; a bad bitch \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" broke bitch cant tell me nothing \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>21</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" cancel that bitch like Nino \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>22</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" cant you see these hoes wont change \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>23</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" fuck no that bitch dont even suck dick \" &amp;#1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>24</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" got ya bitch tip toeing on my hardwood floor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>25</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>\" her pussy lips like Heaven doors \" &amp;#128524;</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" hoe what its hitting for \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>27</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" i met that pussy on Ocean Dr . i gave that p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" i need a trippy bitch who fuck on Hennessy \"</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>29</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>\" i spend my money how i want bitch its my bus...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24753</th>\n",
       "      <td>25266</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you ain't gotta be a dyke to like hoes</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24754</th>\n",
       "      <td>25267</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you are a hoe, hoe, &amp;amp; a hoe.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24755</th>\n",
       "      <td>25268</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you bitches love yall some corny nigga</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24756</th>\n",
       "      <td>25269</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you can masturbate anytime bitch lol &amp;#8220;@g...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24757</th>\n",
       "      <td>25270</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you can never get a group of hoes together wit...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24758</th>\n",
       "      <td>25271</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you can tell when dick recently been in a puss...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24759</th>\n",
       "      <td>25272</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you can't cuff a hoe lmao</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24760</th>\n",
       "      <td>25273</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>you drove me redneck crazy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24761</th>\n",
       "      <td>25274</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you fake niggah lolol</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24762</th>\n",
       "      <td>25275</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you got niggas, and i got bitches.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24763</th>\n",
       "      <td>25276</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>you gotta be a new breed of retarded if you do...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24764</th>\n",
       "      <td>25277</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you gotta understand that these bitches are ch...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24765</th>\n",
       "      <td>25278</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you hoe spice</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24766</th>\n",
       "      <td>25279</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you just want some attention hoe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24767</th>\n",
       "      <td>25280</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>you know what they say, the early bird gets th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24768</th>\n",
       "      <td>25281</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you know what your doing when you favorite a t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24769</th>\n",
       "      <td>25282</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you lil dumb ass bitch, i ain't fuckin wit chu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24770</th>\n",
       "      <td>25283</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you look like AC Green...bitch don't call here...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24771</th>\n",
       "      <td>25284</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you look like your 12 stop talking about fucki...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24772</th>\n",
       "      <td>25285</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you might as well gone pussy pop on a stage</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24773</th>\n",
       "      <td>25286</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you niggers cheat on ya gf's? smh....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24774</th>\n",
       "      <td>25287</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you really care bout dis bitch. my dick all in...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24775</th>\n",
       "      <td>25288</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>you worried bout other bitches, you need me for?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24776</th>\n",
       "      <td>25289</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>you're all niggers</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24777</th>\n",
       "      <td>25290</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>you're such a retard i hope you get type 2 dia...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24778</th>\n",
       "      <td>25291</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>you's a muthaf***in lie &amp;#8220;@LifeAsKing: @2...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24779</th>\n",
       "      <td>25292</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>you've gone and broke the wrong heart baby, an...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24780</th>\n",
       "      <td>25294</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>young buck wanna eat!!.. dat nigguh like I ain...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24781</th>\n",
       "      <td>25295</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>youu got wild bitches tellin you lies</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24782</th>\n",
       "      <td>25296</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>~~Ruffled | Ntac Eileen Dahlia - Beautiful col...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>24783 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       Unnamed: 0  count  hate_speech  offensive_language  neither  class  \\\n",
       "0               0      3            0                   0        3      2   \n",
       "1               1      3            0                   3        0      1   \n",
       "2               2      3            0                   3        0      1   \n",
       "3               3      3            0                   2        1      1   \n",
       "4               4      6            0                   6        0      1   \n",
       "5               5      3            1                   2        0      1   \n",
       "6               6      3            0                   3        0      1   \n",
       "7               7      3            0                   3        0      1   \n",
       "8               8      3            0                   3        0      1   \n",
       "9               9      3            1                   2        0      1   \n",
       "10             10      3            0                   3        0      1   \n",
       "11             11      3            0                   3        0      1   \n",
       "12             12      3            0                   2        1      1   \n",
       "13             13      3            0                   3        0      1   \n",
       "14             14      3            1                   2        0      1   \n",
       "15             15      3            0                   3        0      1   \n",
       "16             16      3            0                   3        0      1   \n",
       "17             17      3            1                   2        0      1   \n",
       "18             18      3            0                   3        0      1   \n",
       "19             19      3            0                   3        0      1   \n",
       "20             20      3            0                   3        0      1   \n",
       "21             21      3            0                   3        0      1   \n",
       "22             22      3            0                   3        0      1   \n",
       "23             23      3            0                   3        0      1   \n",
       "24             24      3            0                   3        0      1   \n",
       "25             25      3            0                   2        1      1   \n",
       "26             26      3            0                   3        0      1   \n",
       "27             27      3            0                   3        0      1   \n",
       "28             28      3            0                   3        0      1   \n",
       "29             29      3            0                   3        0      1   \n",
       "...           ...    ...          ...                 ...      ...    ...   \n",
       "24753       25266      3            1                   2        0      1   \n",
       "24754       25267      3            0                   3        0      1   \n",
       "24755       25268      3            0                   3        0      1   \n",
       "24756       25269      3            0                   3        0      1   \n",
       "24757       25270      3            0                   3        0      1   \n",
       "24758       25271      3            0                   3        0      1   \n",
       "24759       25272      3            0                   3        0      1   \n",
       "24760       25273      3            0                   2        1      1   \n",
       "24761       25274      3            0                   3        0      1   \n",
       "24762       25275      3            1                   2        0      1   \n",
       "24763       25276      3            0                   2        1      1   \n",
       "24764       25277      3            0                   3        0      1   \n",
       "24765       25278      3            0                   3        0      1   \n",
       "24766       25279      3            0                   3        0      1   \n",
       "24767       25280      3            0                   1        2      2   \n",
       "24768       25281      3            0                   3        0      1   \n",
       "24769       25282      3            0                   3        0      1   \n",
       "24770       25283      3            0                   3        0      1   \n",
       "24771       25284      3            0                   3        0      1   \n",
       "24772       25285      3            0                   3        0      1   \n",
       "24773       25286      3            1                   2        0      1   \n",
       "24774       25287      3            0                   3        0      1   \n",
       "24775       25288      3            0                   3        0      1   \n",
       "24776       25289      3            3                   0        0      0   \n",
       "24777       25290      3            2                   1        0      0   \n",
       "24778       25291      3            0                   2        1      1   \n",
       "24779       25292      3            0                   1        2      2   \n",
       "24780       25294      3            0                   3        0      1   \n",
       "24781       25295      6            0                   6        0      1   \n",
       "24782       25296      3            0                   0        3      2   \n",
       "\n",
       "                                                   tweet  \n",
       "0      !!! RT @mayasolovely: As a woman you shouldn't...  \n",
       "1      !!!!! RT @mleew17: boy dats cold...tyga dwn ba...  \n",
       "2      !!!!!!! RT @UrKindOfBrand Dawg!!!! RT @80sbaby...  \n",
       "3      !!!!!!!!! RT @C_G_Anderson: @viva_based she lo...  \n",
       "4      !!!!!!!!!!!!! RT @ShenikaRoberts: The shit you...  \n",
       "5      !!!!!!!!!!!!!!!!!!\"@T_Madison_x: The shit just...  \n",
       "6      !!!!!!\"@__BrighterDays: I can not just sit up ...  \n",
       "7      !!!!&#8220;@selfiequeenbri: cause I'm tired of...  \n",
       "8      \" &amp; you might not get ya bitch back &amp; ...  \n",
       "9      \" @rhythmixx_ :hobbies include: fighting Maria...  \n",
       "10     \" Keeks is a bitch she curves everyone \" lol I...  \n",
       "11                    \" Murda Gang bitch its Gang Land \"  \n",
       "12     \" So hoes that smoke are losers ? \" yea ... go...  \n",
       "13         \" bad bitches is the only thing that i like \"  \n",
       "14                               \" bitch get up off me \"  \n",
       "15                       \" bitch nigga miss me with it \"  \n",
       "16                                \" bitch plz whatever \"  \n",
       "17                             \" bitch who do you love \"  \n",
       "18                    \" bitches get cut off everyday B \"  \n",
       "19                    \" black bottle &amp; a bad bitch \"  \n",
       "20                  \" broke bitch cant tell me nothing \"  \n",
       "21                       \" cancel that bitch like Nino \"  \n",
       "22               \" cant you see these hoes wont change \"  \n",
       "23     \" fuck no that bitch dont even suck dick \" &#1...  \n",
       "24     \" got ya bitch tip toeing on my hardwood floor...  \n",
       "25        \" her pussy lips like Heaven doors \" &#128524;  \n",
       "26                          \" hoe what its hitting for \"  \n",
       "27     \" i met that pussy on Ocean Dr . i gave that p...  \n",
       "28        \" i need a trippy bitch who fuck on Hennessy \"  \n",
       "29     \" i spend my money how i want bitch its my bus...  \n",
       "...                                                  ...  \n",
       "24753             you ain't gotta be a dyke to like hoes  \n",
       "24754                   you are a hoe, hoe, &amp; a hoe.  \n",
       "24755             you bitches love yall some corny nigga  \n",
       "24756  you can masturbate anytime bitch lol &#8220;@g...  \n",
       "24757  you can never get a group of hoes together wit...  \n",
       "24758  you can tell when dick recently been in a puss...  \n",
       "24759                          you can't cuff a hoe lmao  \n",
       "24760                         you drove me redneck crazy  \n",
       "24761                              you fake niggah lolol  \n",
       "24762                 you got niggas, and i got bitches.  \n",
       "24763  you gotta be a new breed of retarded if you do...  \n",
       "24764  you gotta understand that these bitches are ch...  \n",
       "24765                                      you hoe spice  \n",
       "24766                   you just want some attention hoe  \n",
       "24767  you know what they say, the early bird gets th...  \n",
       "24768  you know what your doing when you favorite a t...  \n",
       "24769  you lil dumb ass bitch, i ain't fuckin wit chu...  \n",
       "24770  you look like AC Green...bitch don't call here...  \n",
       "24771  you look like your 12 stop talking about fucki...  \n",
       "24772        you might as well gone pussy pop on a stage  \n",
       "24773              you niggers cheat on ya gf's? smh....  \n",
       "24774  you really care bout dis bitch. my dick all in...  \n",
       "24775   you worried bout other bitches, you need me for?  \n",
       "24776                                 you're all niggers  \n",
       "24777  you're such a retard i hope you get type 2 dia...  \n",
       "24778  you's a muthaf***in lie &#8220;@LifeAsKing: @2...  \n",
       "24779  you've gone and broke the wrong heart baby, an...  \n",
       "24780  young buck wanna eat!!.. dat nigguh like I ain...  \n",
       "24781              youu got wild bitches tellin you lies  \n",
       "24782  ~~Ruffled | Ntac Eileen Dahlia - Beautiful col...  \n",
       "\n",
       "[24783 rows x 7 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>count</th>\n",
       "      <th>hate_speech</th>\n",
       "      <th>offensive_language</th>\n",
       "      <th>neither</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>24783.000000</td>\n",
       "      <td>24783.000000</td>\n",
       "      <td>24783.000000</td>\n",
       "      <td>24783.000000</td>\n",
       "      <td>24783.000000</td>\n",
       "      <td>24783.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>12681.192027</td>\n",
       "      <td>3.243473</td>\n",
       "      <td>0.280515</td>\n",
       "      <td>2.413711</td>\n",
       "      <td>0.549247</td>\n",
       "      <td>1.110277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>7299.553863</td>\n",
       "      <td>0.883060</td>\n",
       "      <td>0.631851</td>\n",
       "      <td>1.399459</td>\n",
       "      <td>1.113299</td>\n",
       "      <td>0.462089</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>6372.500000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>12703.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>18995.500000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>25296.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>9.000000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         Unnamed: 0         count   hate_speech  offensive_language  \\\n",
       "count  24783.000000  24783.000000  24783.000000        24783.000000   \n",
       "mean   12681.192027      3.243473      0.280515            2.413711   \n",
       "std     7299.553863      0.883060      0.631851            1.399459   \n",
       "min        0.000000      3.000000      0.000000            0.000000   \n",
       "25%     6372.500000      3.000000      0.000000            2.000000   \n",
       "50%    12703.000000      3.000000      0.000000            3.000000   \n",
       "75%    18995.500000      3.000000      0.000000            3.000000   \n",
       "max    25296.000000      9.000000      7.000000            9.000000   \n",
       "\n",
       "            neither         class  \n",
       "count  24783.000000  24783.000000  \n",
       "mean       0.549247      1.110277  \n",
       "std        1.113299      0.462089  \n",
       "min        0.000000      0.000000  \n",
       "25%        0.000000      1.000000  \n",
       "50%        0.000000      1.000000  \n",
       "75%        0.000000      1.000000  \n",
       "max        9.000000      2.000000  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Unnamed: 0', 'count', 'hate_speech', 'offensive_language', 'neither',\n",
       "       'class', 'tweet'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Columns key:\n",
    "count = number of CrowdFlower users who coded each tweet (min is 3, sometimes more users coded a tweet when judgments were determined to be unreliable by CF).\n",
    "\n",
    "\n",
    "hate_speech = number of CF users who judged the tweet to be hate speech.\n",
    "\n",
    "\n",
    "offensive_language = number of CF users who judged the tweet to be offensive.\n",
    "\n",
    "\n",
    "neither = number of CF users who judged the tweet to be neither offensive nor non-offensive.\n",
    "\n",
    "\n",
    "class = class label for majority of CF users.\n",
    "\n",
    "    0 - hate speech\n",
    "    1 - offensive  language\n",
    "    2 - neither\n",
    "\n",
    "tweet = raw tweet text\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1108a1208>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD5CAYAAAAuneICAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFltJREFUeJzt3X+MVeWdx/H3MKOMtHfomAx1mzQhuu0nplmtwQWs0E6U\nXQKtsWvXrEtsad1aZd1qfyRqBde6oemqlS6QFVeQiEW3plrTLl0EU12FadEUa4Jd81VQ02ZTm1kz\nzEx3AgjO/nHOtDfT4f7kzpnyfF7JJOc89znnfs/D4XzuOffce9tGR0cxM7P0TCu6ADMzK4YDwMws\nUQ4AM7NEOQDMzBLlADAzS5QDwMwsUR2VHpR0CrAZmA1MB1YD/w08AIwCLwHXRcQ7kq4GrgGOAqsj\nYpuk04CtwCxgGFgeEf2S5gNr8747I+L2FmybmZlV0FbpcwCSPgecGxFfknQ68GL+tyYi/kvSvcAO\n4KfAk8D5QCewO5++DuiKiK9LugK4ICJukPQi8CngNeBHwMqI+HmlQvv7h5v6wEJ39wwGBkaaWUVL\nuK76uK76uK76nIx19fSU2o73WLVLQN8Dbs2n28hesc8BnsnbtgOLgLlAX0QcjohBYD9wDrAAeKK8\nr6QuYHpEHIiIUbIAWVT3VtWpo6O91U/RENdVH9dVH9dVn9TqqngJKCJ+CyCpBDwKrAK+lR+4Ibus\nMxPoAgbLFp2ovbxtaFzfM6sV2t09o+lB6OkpNbV8q7iu+riu+riu+qRUV8UAAJD0fuBx4J6IeFjS\nnWUPl4CDZAf0UpX2an0rava0rKenRH//cFPraAXXVR/XVR/XVZ+Tsa5KwVHxEpCk9wI7gZsiYnPe\n/HNJvfn0EmAX8DywUFKnpJnA2WRvEPcBS8v7RsQQcETSWZLagMX5OszMbBJVOwO4BegGbpU09l7A\nDcA6SacCLwOPRsQxSevIDuTTyN7UPSRpA7BF0m7gCLAsX8e1wENAO9ldQM+d0K0yM7OqKt4FNJU0\nexfQyXhq10quqz6uqz6uqz5NXgJq+C4gMzM7STkAzMwS5QAwM0uUA8DMLFFVPwdgZtVd8tUfFPK8\nm2++qJDntZODzwDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLl\nADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS1RNXwctaR5wR0T0SvoucEb+0GxgT0RcIWktsAAY++HK\nS8l+CH4rMCtvXx4R/ZLmA2uBo2Q/Cn/7idogMzOrTdUzAEk3ApuAToCIuCIieoG/Ag4CX867zgEW\nR0Rv/jcIrAD2RcRC4EFgVd73XmAZWWDMk3TeidskMzOrRS2XgA4Al03QfjuwPiJ+LWka8AHgPkl9\nkq7K+ywAnsintwOLJHUB0yPiQESMAjuARU1thZmZ1a3qJaCIeEzS7PI2SbOAi/n9q/93AeuBNUA7\n8LSknwFdwGDeZxiYmbcNla1uGDizWh3d3TPo6Giv1q2inp5SU8u3iuuqz1Stqwi1jMVUHS/XVZ9W\n1NXoT0L+NfBwRBzL50eAtRExAiDpKeBcsgP9WNUlsktG5W3l7RUNDIw0WGqmp6dEf/9w9Y6TzHXV\nZ6rWVZRqYzFVx8t11aeZuioFR6N3AS0iu6Qz5oNAn6R2SaeQXfp5AegDluZ9lgC7ImIIOCLpLElt\nwGJgV4N1mJlZgxo9AxDw2thMRLws6TvAHuBt4MGI+IWk14EtknaT3RG0LF/kWuAhsstFOyPiuUY3\nwMzMGlNTAETEG8D8svkPTdDnLuCucW0jwOUT9N1Tvj4zM5t8/iCYmVmiHABmZolyAJiZJcoBYGaW\nKAeAmVmiHABmZolyAJiZJcoBYGaWKAeAmVmiHABmZolyAJiZJcoBYGaWKAeAmVmiHABmZolyAJiZ\nJcoBYGaWKAeAmVmiHABmZolyAJiZJaqm3wSWNA+4IyJ6JZ0HbANezR/eEBGPSLoauAY4CqyOiG2S\nTgO2ArOAYWB5RPRLmg+szfvujIjbT+xmmZlZNVXPACTdCGwCOvOmOcCaiOjN/x6RdAZwPXAhsBj4\npqTpwApgX0QsBB4EVuXruBdYBiwA5uWhYmZmk6iWS0AHgMvK5ucAH5f0rKT7JZWAuUBfRByOiEFg\nP3AO2QH+iXy57cAiSV3A9Ig4EBGjwA5g0QnaHjMzq1HVS0AR8Zik2WVNzwObImKvpJXAbcCLwGBZ\nn2FgJtBV1l7eNjSu75nV6ujunkFHR3u1bhX19JSaWr5VXFd9pmpdRahlLKbqeLmu+rSirpreAxjn\n8Yg4ODYNrAeeBcqrKwEHyQ70pQpt5e0VDQyMNFDq7/X0lOjvH25qHa3guuozVesqSrWxmKrj5brq\n00xdlYKjkbuAdkiam09fDOwlOytYKKlT0kzgbOAloA9YmvddAuyKiCHgiKSzJLWRvWewq4E6zMys\nCY2cAawA1kt6G3gT+EJEDElaR3YgnwasjIhDkjYAWyTtBo6QvfELcC3wENBOdhfQc81uiJmZ1aem\nAIiIN4D5+fQLZHf7jO+zEdg4rm0EuHyCvnvG1mdmZsXwB8HMzBLlADAzS5QDwMwsUQ4AM7NEOQDM\nzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4A\nM7NEOQDMzBLlADAzS5QDwMwsUTX9JrCkecAdEdEr6cPAeuAYcBj4TET8RtJaYAEwnC92KdkPwW8F\nZuXtyyOiX9J8YC1wlOxH4W8/kRtlZmbVVT0DkHQjsAnozJvWAl+MiF7g+8BNefscYHFE9OZ/g8AK\nYF9ELAQeBFblfe8FlpEFxjxJ552g7TEzsxrVcgnoAHBZ2fwVEfFiPt0BHJI0DfgAcJ+kPklX5Y8v\nAJ7Ip7cDiyR1AdMj4kBEjAI7gEXNboiZmdWn6iWgiHhM0uyy+V8DSPoI8A/AR4F3kV0WWgO0A09L\n+hnQBQzmiw4DM/O2obKnGAbOrFZHd/cMOjraq29RBT09paaWbxXXVZ+pWlcRahmLqTperqs+rair\npvcAxpP0N8BK4OP5Nf12YG1EjOSPPwWcS3agH6u6BBwc11beXtHAwEgjpf5OT0+J/v7h6h0nmeuq\nz1StqyjVxmKqjpfrqk8zdVUKjrrvApJ0Jdkr/96IeC1v/iDQJ6ld0ilkl35eAPqApXmfJcCuiBgC\njkg6S1IbsBjYVW8dZmbWnLrOAPJX+uuAXwLflwTwTETcJuk7wB7gbeDBiPiFpNeBLZJ2k90RtCxf\n1bXAQ2SXi3ZGxHMnZGvMzKxmNQVARLwBzM9nTz9On7uAu8a1jQCXT9B3T9n6zMysAP4gmJlZohwA\nZmaJcgCYmSXKAWBmligHgJlZohwAZmaJcgCYmSXKAWBmligHgJlZohwAZmaJcgCYmSXKAWBmligH\ngJlZohwAZmaJcgCYmSXKAWBmligHgJlZohwAZmaJcgCYmSWqpt8EljQPuCMieiX9KfAAMAq8BFwX\nEe9Iuhq4BjgKrI6IbZJOA7YCs4BhYHlE9EuaD6zN++6MiNtP9IaZmVllVc8AJN0IbAI686Y1wKqI\nWAi0AZdKOgO4HrgQWAx8U9J0YAWwL+/7ILAqX8e9wDJgATBP0nknbpPMzKwWtVwCOgBcVjY/B3gm\nn94OLALmAn0RcTgiBoH9wDlkB/gnyvtK6gKmR8SBiBgFduTrMDOzSVT1ElBEPCZpdllTW37ghuyy\nzkygCxgs6zNRe3nb0Li+Z1aro7t7Bh0d7dW6VdTTU2pq+VZxXfWZqnUVoZaxmKrj5brq04q6anoP\nYJx3yqZLwEGyA3qpSnu1vhUNDIw0UOrv9fSU6O8fbmodreC66jNV6ypKtbGYquPluurTTF2VgqOR\nu4B+Lqk3n14C7AKeBxZK6pQ0Ezib7A3iPmBped+IGAKOSDpLUhvZewa7GqjDzMya0MgZwFeBjZJO\nBV4GHo2IY5LWkR3IpwErI+KQpA3AFkm7gSNkb/wCXAs8BLST3QX0XLMbYmZm9akpACLiDWB+Pv0K\n8LEJ+mwENo5rGwEun6DvnrH1mZlZMfxBMDOzRDkAzMwS5QAwM0uUA8DMLFEOADOzRDkAzMwS5QAw\nM0uUA8DMLFEOADOzRDkAzMwS5QAwM0uUA8DMLFEOADOzRDkAzMwS5QAwM0uUA8DMLFEOADOzRDkA\nzMwS5QAwM0tUIz8Kj6TPAp/NZzuBDwMXANuAV/P2DRHxiKSrgWuAo8DqiNgm6TRgKzALGAaWR0R/\noxthZmb1aygAIuIB4AEASf8KbAbmAGsi4u6xfpLOAK4HzicLit2SngRWAPsi4uuSrgBWATc0vhlm\nZlavpi4BSTof+FBE3EcWAB+X9Kyk+yWVgLlAX0QcjohBYD9wDrAAeCJfzXZgUTN1mJlZ/Ro6Ayhz\nC3B7Pv08sCki9kpaCdwGvAgMlvUfBmYCXWXtY20VdXfPoKOjvalie3pKTS3fKq6rPlO1riLUMhZT\ndbxcV31aUVfDASDpPYAi4um86fGIODg2DawHngXKqy4BB4GhsvaxtooGBkYaLRXIBq+/f7ipdbSC\n66rPVK2rKNXGYqqOl+uqTzN1VQqOZi4BfRT4cdn8Dklz8+mLgb1kZwULJXVKmgmcDbwE9AFL875L\ngF1N1GFmZg1o5hKQgNfK5lcA6yW9DbwJfCEihiStIzvATwNWRsQhSRuALZJ2A0eAZU3UYWZmDWg4\nACLirnHzLwAXTtBvI7BxXNsIcHmjz21mZs3zB8HMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAz\nS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDM\nzBLlADAzS5QDwMwsUQ3/JrCkF4ChfPZ14BvAA8Ao8BJwXUS8I+lq4BrgKLA6IrZJOg3YCswChoHl\nEdHf8FaYmVndGjoDkNQJtEVEb/73OWANsCoiFgJtwKWSzgCuJ/ux+MXANyVNB1YA+/K+DwKrTsC2\nmJlZHRo9AzgXmCFpZ76OW4A5wDP549uBvwSOAX0RcRg4LGk/cA6wALizrO+tDdZhZmYNajQARoBv\nAZuAD5AdxNsiYjR/fBiYCXQBg2XLTdQ+1lZRd/cMOjraGyw309NTamr5VnFd9ZmqdRWhlrGYquPl\nuurTiroaDYBXgP35Af8VSW+RnQGMKQEHyd4jKFVpH2uraGBgpMFSMz09Jfr7h5taRyu4rvpM1bqK\nUm0spup4ua76NFNXpeBo9C6gq4C7ASS9j+wV/U5JvfnjS4BdwPPAQkmdkmYCZ5O9QdwHLB3X18zM\nJlGjZwD3Aw9I2k12189VwP8CGyWdCrwMPBoRxyStIzvATwNWRsQhSRuALfnyR4BlzW6ImZnVp6EA\niIjjHbQ/NkHfjcDGcW0jwOWNPLeZmZ0Y/iCYmVmiHABmZolyAJiZJcoBYGaWqIa/C8jMLCVX/fNT\nhT33f9x9aUvW6zMAM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NE\nOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLV0NdBSzoF2AzMBqYDq4FfAduAV/NuGyLiEUlXA9cA\nR4HVEbFN0mnAVmAWMAwsj4j+ZjbEzMzq0+jvAVwJvBURn5Z0OvAi8E/Amoi4e6yTpDOA64HzgU5g\nt6QngRXAvoj4uqQrgFXADU1sh5mZ1anRAPge8Gg+3Ub26n4OIEmXkp0FfAmYC/RFxGHgsKT9wDnA\nAuDOfPntwK0N1mFmZg1qKAAi4rcAkkpkQbCK7FLQpojYK2klcBvZmcFg2aLDwEygq6x9rK2i7u4Z\ndHS0N1Lu7/T0lJpavlVcV32mal1FqGUspup4ua76tKKuhn8SUtL7gceBeyLiYUnviYiD+cOPA+uB\nZ4HyqkvAQWCorH2sraKBgZFGSwWywevvH25qHa3guuozVesqSrWxmKrj5brq12hdlYKjobuAJL0X\n2AncFBGb8+Ydkubm0xcDe4HngYWSOiXNBM4GXgL6gKV53yXArkbqMDOzxjV6BnAL0A3cKmns+v1X\ngG9Leht4E/hCRAxJWkd2gJ8GrIyIQ5I2AFsk7QaOAMua2gozM6tbo+8B3MDEd+1cOEHfjcDGcW0j\nwOWNPLeZmZ0Y/iCYmVmiHABmZolyAJiZJcoBYGaWKAeAmVmiHABmZolq+JPAf2wu+eoPCnnezTdf\nVMjzmplV4zMAM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDM\nzBLlADAzS5QDwMwsUYV9GZykacA9wLnAYeDzEbG/qHrMzFJT5BnAJ4HOiLgAuBm4u8BazMySU2QA\nLACeAIiIPcD5BdZiZpacttHR0UKeWNIm4LGI2J7P/xI4MyKOFlKQmVliijwDGAJKZfPTfPA3M5s8\nRQZAH7AUQNJ8YF+BtZiZJafIn4R8HPgLST8B2oDPFViLmVlyCnsPwMzMiuUPgpmZJcoBYGaWKAeA\nmVmiinwT+ISo9pUSki4B/hE4CmyOiI2T8TUUNdT1t8CX8rr2AX8fEe9IeoHsFlmA1yPihL45XkNd\nXwY+D/TnTdcAr1ZaptV1SToD+G5Z9w8DN0fEva0er7L65gF3RETvuPZC9q8a6ipk/6qhrkL2r0p1\nFbl/SToF2AzMBqYDqyPih2WPt3T/+qMPAMq+UiK/nfRu4FL43eB+G/hz4P+APkk/BC483jKTVNdp\nwGrgzyJiRNK/A5+QtBNoG/+fZrLqys0BPhMRe8caJF1WZZmW1hURbwK9eS0XAN8ANkrqpPXjhaQb\ngU+T7UPl7UXuX5XqKnL/Om5duaL2r+PWVfD+dSXwVkR8WtLpwIvAD/NaWr5/nQyXgCp9pcTZwP6I\nGIiII8Bu4KNVlpmMug4DH4mIkXy+AzhElugzJO2U9FT+jzuZdUH2H/RrknZL+lqNy0xGXUhqA9YD\nKyLiGJMzXgAHgMsmaC9y/6pUV5H7V6W6oLj9q1pdRe1f3wNuzafbyF7pj2n5/nUyBEAXMFg2f0xS\nx3EeGwZmVlmm5XVFxDsR8RsASV8E3g08CYwA3wIWA9cCD01mXbnv5s99EbBA0idqWGYy6gK4BPhF\nREQ+PxnjRUQ8Brw9wUNF7l/Hravg/avSeEFx+1e1uqCA/SsifhsRw5JKwKPAqrKHW75/nQyXgCp9\npcT4x0rAwSrLTEZdY9e87wQ+CHwqIkYlvUKW+KPAK5LeAv4E+NVk1JW/AvqXiBjM538EnFdtW1pd\nV5krgbVl85MxXpUUuX9VVOD+VammIvevWhSyf0l6P9kHY++JiIfLHmr5/nUynAFU+kqJl4EPSDpd\n0qlkp08/rbLMZNQF8G9AJ/DJslP1q8i/FlvS+8iS/teTWFcX8JKkd+f/WS8C9tawLa2ua8z5wE/K\n5idjvCopcv+qpqj9q5Ii969aTPr+Jem9wE7gpojYPO7hlu9fJ8MZwB98pYSkZcC7I+I+SV8BdpCF\n3eaI+B9Jk/E1FMetC/gZ8HfALuApSZC98rgfeEDSbmAUuKoFr4SqjdctwNNk15F/HBH/mb+aLGy8\n8rp6gKH81diYyRivPzBF9q/j1kWx+9dx6yp4/6pWV1H71y1AN3CrpLH3AjYC75qM/ctfBWFmlqiT\n4RKQmZk1wAFgZpYoB4CZWaIcAGZmiXIAmJklygFgZpYoB4CZWaL+H1ff6+tI9kgAAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x10f2de780>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['class'].hist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This histogram shows the imbalanced nature of the task - most tweets containing \"hate\" words as defined by Hatebase were \n",
    "only considered to be offensive by the CF coders. More tweets were considered to be neither hate speech nor offensive language than were considered hate speech."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tweets=df.tweet"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature generation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stopwords=stopwords = nltk.corpus.stopwords.words(\"english\")\n",
    "\n",
    "other_exclusions = [\"#ff\", \"ff\", \"rt\"]\n",
    "stopwords.extend(other_exclusions)\n",
    "\n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "\n",
    "def preprocess(text_string):\n",
    "    \"\"\"\n",
    "    Accepts a text string and replaces:\n",
    "    1) urls with URLHERE\n",
    "    2) lots of whitespace with one instance\n",
    "    3) mentions with MENTIONHERE\n",
    "\n",
    "    This allows us to get standardized counts of urls and mentions\n",
    "    Without caring about specific people mentioned\n",
    "    \"\"\"\n",
    "    space_pattern = '\\s+'\n",
    "    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'\n",
    "        '[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
    "    mention_regex = '@[\\w\\-]+'\n",
    "    parsed_text = re.sub(space_pattern, ' ', text_string)\n",
    "    parsed_text = re.sub(giant_url_regex, '', parsed_text)\n",
    "    parsed_text = re.sub(mention_regex, '', parsed_text)\n",
    "    return parsed_text\n",
    "\n",
    "def tokenize(tweet):\n",
    "    \"\"\"Removes punctuation & excess whitespace, sets to lowercase,\n",
    "    and stems tweets. Returns a list of stemmed tokens.\"\"\"\n",
    "    tweet = \" \".join(re.split(\"[^a-zA-Z]*\", tweet.lower())).strip()\n",
    "    tokens = [stemmer.stem(t) for t in tweet.split()]\n",
    "    return tokens\n",
    "\n",
    "def basic_tokenize(tweet):\n",
    "    \"\"\"Same as tokenize but without the stemming\"\"\"\n",
    "    tweet = \" \".join(re.split(\"[^a-zA-Z.,!?]*\", tweet.lower())).strip()\n",
    "    return tweet.split()\n",
    "\n",
    "vectorizer = TfidfVectorizer(\n",
    "    tokenizer=tokenize,\n",
    "    preprocessor=preprocess,\n",
    "    ngram_range=(1, 3),\n",
    "    stop_words=stopwords,\n",
    "    use_idf=True,\n",
    "    smooth_idf=False,\n",
    "    norm=None,\n",
    "    decode_error='replace',\n",
    "    max_features=10000,\n",
    "    min_df=5,\n",
    "    max_df=0.75\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import warnings\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Construct tfidf matrix and get relevant scores\n",
    "tfidf = vectorizer.fit_transform(tweets).toarray()\n",
    "vocab = {v:i for i, v in enumerate(vectorizer.get_feature_names())}\n",
    "idf_vals = vectorizer.idf_\n",
    "idf_dict = {i:idf_vals[i] for i in vocab.values()} #keys are indices; values are IDF scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Get POS tags for tweets and save as a string\n",
    "tweet_tags = []\n",
    "for t in tweets:\n",
    "    tokens = basic_tokenize(preprocess(t))\n",
    "    tags = nltk.pos_tag(tokens)\n",
    "    tag_list = [x[1] for x in tags]\n",
    "    tag_str = \" \".join(tag_list)\n",
    "    tweet_tags.append(tag_str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#We can use the TFIDF vectorizer to get a token matrix for the POS tags\n",
    "pos_vectorizer = TfidfVectorizer(\n",
    "    tokenizer=None,\n",
    "    lowercase=False,\n",
    "    preprocessor=None,\n",
    "    ngram_range=(1, 3),\n",
    "    stop_words=None,\n",
    "    use_idf=False,\n",
    "    smooth_idf=False,\n",
    "    norm=None,\n",
    "    decode_error='replace',\n",
    "    max_features=5000,\n",
    "    min_df=5,\n",
    "    max_df=0.75,\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Construct POS TF matrix and get vocab dict\n",
    "pos = pos_vectorizer.fit_transform(pd.Series(tweet_tags)).toarray()\n",
    "pos_vocab = {v:i for i, v in enumerate(pos_vectorizer.get_feature_names())}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Now get other features\n",
    "sentiment_analyzer = VS()\n",
    "\n",
    "def count_twitter_objs(text_string):\n",
    "    \"\"\"\n",
    "    Accepts a text string and replaces:\n",
    "    1) urls with URLHERE\n",
    "    2) lots of whitespace with one instance\n",
    "    3) mentions with MENTIONHERE\n",
    "    4) hashtags with HASHTAGHERE\n",
    "\n",
    "    This allows us to get standardized counts of urls and mentions\n",
    "    Without caring about specific people mentioned.\n",
    "    \n",
    "    Returns counts of urls, mentions, and hashtags.\n",
    "    \"\"\"\n",
    "    space_pattern = '\\s+'\n",
    "    giant_url_regex = ('http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|'\n",
    "        '[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')\n",
    "    mention_regex = '@[\\w\\-]+'\n",
    "    hashtag_regex = '#[\\w\\-]+'\n",
    "    parsed_text = re.sub(space_pattern, ' ', text_string)\n",
    "    parsed_text = re.sub(giant_url_regex, 'URLHERE', parsed_text)\n",
    "    parsed_text = re.sub(mention_regex, 'MENTIONHERE', parsed_text)\n",
    "    parsed_text = re.sub(hashtag_regex, 'HASHTAGHERE', parsed_text)\n",
    "    return(parsed_text.count('URLHERE'),parsed_text.count('MENTIONHERE'),parsed_text.count('HASHTAGHERE'))\n",
    "\n",
    "def other_features(tweet):\n",
    "    \"\"\"This function takes a string and returns a list of features.\n",
    "    These include Sentiment scores, Text and Readability scores,\n",
    "    as well as Twitter specific features\"\"\"\n",
    "    sentiment = sentiment_analyzer.polarity_scores(tweet)\n",
    "    \n",
    "    words = preprocess(tweet) #Get text only\n",
    "    \n",
    "    syllables = textstat.syllable_count(words)\n",
    "    num_chars = sum(len(w) for w in words)\n",
    "    num_chars_total = len(tweet)\n",
    "    num_terms = len(tweet.split())\n",
    "    num_words = len(words.split())\n",
    "    avg_syl = round(float((syllables+0.001))/float(num_words+0.001),4)\n",
    "    num_unique_terms = len(set(words.split()))\n",
    "    \n",
    "    ###Modified FK grade, where avg words per sentence is just num words/1\n",
    "    FKRA = round(float(0.39 * float(num_words)/1.0) + float(11.8 * avg_syl) - 15.59,1)\n",
    "    ##Modified FRE score, where sentence fixed to 1\n",
    "    FRE = round(206.835 - 1.015*(float(num_words)/1.0) - (84.6*float(avg_syl)),2)\n",
    "    \n",
    "    twitter_objs = count_twitter_objs(tweet)\n",
    "    retweet = 0\n",
    "    if \"rt\" in words:\n",
    "        retweet = 1\n",
    "    features = [FKRA, FRE,syllables, avg_syl, num_chars, num_chars_total, num_terms, num_words,\n",
    "                num_unique_terms, sentiment['neg'], sentiment['pos'], sentiment['neu'], sentiment['compound'],\n",
    "                twitter_objs[2], twitter_objs[1],\n",
    "                twitter_objs[0], retweet]\n",
    "    #features = pandas.DataFrame(features)\n",
    "    return features\n",
    "\n",
    "def get_feature_array(tweets):\n",
    "    feats=[]\n",
    "    for t in tweets:\n",
    "        feats.append(other_features(t))\n",
    "    return np.array(feats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "other_features_names = [\"FKRA\", \"FRE\",\"num_syllables\", \"avg_syl_per_word\", \"num_chars\", \"num_chars_total\", \\\n",
    "                        \"num_terms\", \"num_words\", \"num_unique_words\", \"vader neg\",\"vader pos\",\"vader neu\", \\\n",
    "                        \"vader compound\", \"num_hashtags\", \"num_mentions\", \"num_urls\", \"is_retweet\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "feats = get_feature_array(tweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Now join them all up\n",
    "M = np.concatenate([tfidf,pos,feats],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24783, 11172)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "M.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Finally get a list of variable names\n",
    "variables = ['']*len(vocab)\n",
    "for k,v in vocab.items():\n",
    "    variables[v] = k\n",
    "\n",
    "pos_variables = ['']*len(pos_vocab)\n",
    "for k,v in pos_vocab.items():\n",
    "    pos_variables[v] = k\n",
    "\n",
    "feature_names = variables+pos_variables+other_features_names"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running the model\n",
    "\n",
    "The best model was selected using a GridSearch with 5-fold CV."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X = pd.DataFrame(M)\n",
    "y = df['class'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42, test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import StratifiedKFold, GridSearchCV\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pipe = Pipeline(\n",
    "        [('select', SelectFromModel(LogisticRegression(class_weight='balanced',\n",
    "                                                  penalty=\"l1\", C=0.01))),\n",
    "        ('model', LogisticRegression(class_weight='balanced',penalty='l2'))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "param_grid = [{}] # Optionally add parameters here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "grid_search = GridSearchCV(pipe, \n",
    "                           param_grid,\n",
    "                           cv=StratifiedKFold(n_splits=5, \n",
    "                                              random_state=42).split(X_train, y_train), \n",
    "                           verbose=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 1 candidates, totalling 5 fits\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   8.0s\n",
      "[CV]  ................................................................\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    8.4s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV] ................................................. , total=   4.7s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   4.8s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   5.6s\n",
      "[CV]  ................................................................\n",
      "[CV] ................................................. , total=   4.1s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed:   29.2s finished\n"
     ]
    }
   ],
   "source": [
    "model = grid_search.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_preds = model.predict(X_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "report = classification_report( y_test, y_preds )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "***Note: Results in paper are from best model retrained on the entire dataset (see the other notebook). Here the results are reported after using cross-validation and only for the held-out set.***"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "          0       0.44      0.59      0.51       164\n",
      "          1       0.96      0.91      0.93      1905\n",
      "          2       0.83      0.94      0.88       410\n",
      "\n",
      "avg / total       0.91      0.89      0.90      2479\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(report)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAUEAAAFBCAYAAADpDh0xAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FdX5x/FPQlA22UTZcSnyVFErogJ1BReURStaqVtd\nsVgV/bnUWlREse5aQVTAtbYCoijFKrgUFBCwrYCtyCNtRUEF2USIsiX398eZhCQm4SYkk4T5vl+v\nvLh37pmZM8PNk3PmnHkmI5VKISKSVJlVXQERkaqkICgiiaYgKCKJpiAoIommICgiiaYgKCKJllXV\nFUjXJZdcork8Mejdu3dVVyExevToUdVVSIzGjRtnlPSZWoIikmgKgiKSaAqCIpJoCoIikmgKgiKS\naAqCIpJoCoIikmgKgiKSaAqCIpJoCoIikmgKgiKSaAqCIpJoCoIikmgKgiKSaAqCIpJoCoIikmgK\ngiKSaAqCIpJoCoIikmgKgiKSaAqCIpJoCoIikmgKgiKSaAqCIpJoCoIikmgKgiKSaAqCIpJoCoIi\nkmgKgiKSaAqCIpJoCoIikmgKgiKSaAqCIpJoCoIikmgKgiKSaAqCIpJoCoIikmgKgiKSaAqCIpJo\nCoIikmhZVV2Bmujggw+mX79+1K5dm2XLlvH000+zcePGQmXOOussDjvsMLKzswFYvnw5o0aNon79\n+px33nm0a9eOTZs2MXPmTP72t79VxWHUCIsWLWLq1Kls3bqVFi1acMYZZ1CnTp1CZd577z3mzp1L\nRkYGTZs2pV+/fjRo0IAtW7YwadIkli1bRiqVom3btpx22mnUrl27io6meps5cyaPPfYYmzdvpn37\n9gwePJgGDRqkVWbdunXce++9fPLJJ9StW5c+ffpw1llnVdGRlI1agmXUoEEDLrroIh599FEGDx7M\nypUrOfPMM39Qrn379owaNYqhQ4cydOhQRo0aBUD//v3ZtGkTN998M3feeScHHXQQBx98cNyHUSNs\n2LCBF198kXPPPZfrrruOpk2bMmXKlEJlvvjiC2bMmMHll1/ONddcQ7NmzXjzzTcBmDZtGrm5uQwa\nNIirr76aLVu2MH369Co4kupv7dq1DBs2jLvuuosJEybQunVrHn300bTL/OEPf6Bu3bqMGzeOJ598\nkvfee4+ZM2dWxaGUmYJgGXXs2JElS5bw9ddfA+EXrUuXLoXKZGVl0a5dO3r27Mltt93Gr3/9a5o2\nbQrA3nvvzezZs0mlUuTk5PDhhx9y2GGHxX4cNcHixYtp06YNzZo1A6Br167Mnz+fVCqVX6Z169Zc\nf/311KlThy1btrBu3Trq1asHhHPdvXt3MjMzyczMpFWrVqxdu7ZKjqW6mzt3Lvvvvz/t2rUDoF+/\nfkyZMqXQuS6tzKJFizjllFOoVasWtWvX5sgjj6wxPZxYu8Nm1gC4B/gx8HPgLuA6d98QZz12RNOm\nTVmzZk3++7Vr11KvXj3q1KmT3yVu3LgxH3/8MS+99BIrVqygZ8+eXHXVVQwdOpT//e9/dOvWjf/8\n5z9kZWXRuXNncnJyqupwqrV169bRqFGj/PcNGzZk06ZNbNq0qVCXuFatWnz00UdMnDiRrKwsTjzx\nRAA6dOiQX2bt2rXMmjWL008/Pb4DqEFWrFhB8+bN89/vueeeZGdnk52dnd8lLq1Mx44def311/nJ\nT37C5s2bmTZtGllZNeNqW9wtweHAN0BzYCPQEBgdcx12SEZGRrHLc3Nz81+vWrWKhx9+mBUrVgAw\ndepU9thjD5o1a8b48eNJpVIMGTKEK664goULF7J169ZY6l7TFGyFFJSZ+cOvbceOHbnllls4/vjj\neeqppwr9f3zxxReMGjWKbt26sf/++1dafWuyks51rVq10ipz9dVXk5GRwfnnn8+NN97IEUccUWOu\nvcYdBDu5+2Bgi7t/B5wLHBJzHXbImjVrCrVOmjRpQnZ2Nps3b85f1qZNG7p161ZovYyMDHJycqhb\nty4vvvgit956Kw8++CCpVCq/ay2FNW7cmPXr1+e///bbb6lbty677LJL/rJVq1axZMmS/PeHHXYY\n33zzDd9//z0ACxYs4Mknn+Tkk0+me/fusdW9pmnevDmrVq3Kf79y5UoaNmxI3bp10yqTnZ3NlVde\nydixYxkxYgSZmZm0adMm1mMor7iDYNF+Xy0gt7iC1dVHH33Evvvuy5577gnAsccey7x58wqVSaVS\nnH322fnXsrp3787SpUtZu3Ytxx13HKeddhoQundHH300c+fOjfcgaoj99tuPpUuX5v/izZ07lwMO\nOKBQmfXr1zN27Nj8Ufj58+fTvHlz6tevz7/+9S8mT57MxRdfzCGH1Ki/tbHr0qUL//73v/n8888B\nmDhxIkcffXTaZSZOnMjo0aFTt3r1aiZNmkTPnj1jPILyyyipiVsZzOxBYAtwGnA1cCXwmbtfub11\nL7nkkvgquh0HHXQQZ5xxBrVq1WLlypU8+eSTNGvWjAsvvJChQ4cC4SJ+r169yMjIYO3atTzzzDOs\nWbOGOnXqcOmll+YH0ddee405c+ZU5eEU0rt376quQiF5U2RycnJo2rQpZ511FmvWrGHixIkMGjQI\ngDlz5jBnzhwyMzPZbbfdOO2002jatCn3338/GzdupGHDhvnb22uvvfL/CFW1Hj16VHUVCpk1axaP\nPvooW7dupXXr1gwZMoQvv/ySO++8kz/96U8llmnUqBHZ2dncdttt+dORLrjgAk455ZQqPqJtGjdu\nXPx1LOIPgrWB3wJ9CK3AqcDt7r5pe+tWpyC4M6tuQXBnVt2C4M6stCAY9/BNL3e/A7gjb4GZnQ88\nF3M9RESAmIKgmfUFagP3mVkmkBeVawO/R0FQRKpIXC3BQ4AewJ7AoALLtwL3xVQHEZEfiCUI5nWB\nzezX7v7odlcQEYlJ3NcEnzCz04EGhC5xLaB9NHdQRCR2cQfB8cC+QEtgHtAFmB5zHURE8sU9WfoQ\noDMwCbgG+CnQOOY6iIjkizsIfunuW4FPgAPdfSFQL+Y6iIjkizsIZpvZOcAC4CwzOwjYPeY6iIjk\nizsIXknoEr9JuGf4XeD+mOsgIpIv1oERd/8E+E30tn+c+xYRKU5cd4xMBkq899fdT42jHiIiRcXV\nEnyxwOvbgVtj2q+ISKniumPk2bzXZnZNwfciIlWpKh60pJRYIlJt6GlzIpJocQ2MNC3wtpaZNWFb\nOi3cfc0P1xIRqXxxDYysInSD8wLf6gKfpQiJFEREYhfXwIi63SJSLSk4iUiiKQiKSKIpCIpIoikI\nikiiKQiKSKIpCIpIoikIikiiKQiKSKIpCIpIoikIikiiKQiKSKIpCIpIoikIikiiKQiKSKIpCIpI\noikIikiiKQiKSKIpCIpIoikIikiiKQiKSKKl/aAlM+sArHD3dWZ2AnA68A93f7rSaiciUsnSagma\n2YXAR0AnMzsYmAx0AO42s1srr3oiIpUrI5VKbbeQmTnwe3d/1sweBI5x98PMrDvwR3dvW9kV3bhx\n4/YrKjts9913r+oqJMaGDRuqugqJkZGRkVHSZ+leE2wH/C163Rf4S/T6U6Bp+asmIlK10r0m+D/g\nRDP7EvgR24LgBcCiyqiYiEgc0g2CQ4A/R+Unuft8M7sfGEgYIBERqZHSuiYIYGZ7AK3dfX703oBv\n3H1FJdYvn64JxkPXBOOja4LxKe2aYNpTZID1QEczOx14GGgFrNvBuomIVKl0p8i0Bxy4HbgJaEzo\nCn9kZp0rr3oiIpUr3dHh4cAkoD2wKVp2NvAi8IdKqJeISCzSDYI/BR5x9/zrcu6eC9wLdKqMiomI\nxCHdILgBaFnM8gOBtRVXHRGReKUbBB8HRpnZaUAGsL+ZDQBGA09WVuVERCpbWqPD7j7MzNYBI4B6\nhHuHvwYeAO6vvOqJiFSutOcJ5jGz+kCWu8c6PUbzBOOheYLx0TzB+JRrnqCZXQz82d03Ra+Lfp7/\n2t2f2tFKiohUhdK6w7cQpsVsil6XJAUoCIpIjVRiEHT3fQq8HQjMcne130Vkp5Lu6PDzwF6VWRER\nkaqQbhBcABxTmRUREakK6SZQWA88YmZDgSXAxoIfursCpIjUSOkGwQ+iHxGRnUp55gk2BWq5+8rK\nqVLxNE8wHponGB/NE4xPRTxjBDO7zsxWACuB5Wa2Muoei4jUWOnmExwC3AjcChwCdCak3L/czG6q\nvOqJiFSudK8JDgAucffJBZbNN7NlhPuJ76rwmomIxCDd7vBuwCfFLP8E2KPiqiMiEq90g+B7wPVm\nVitvQfT6emBuZVRMRCQO6XaHrwXeBU4ys3nRskOBWsDJlVExEZE4lOWRm7sD5wI/Br4nPHT9eXfP\nrrzqbaMpMvHQFJn4aIpMfEqbIlPmeYJVRUEwHgqC8VEQjM8OP3fYzD4lpMwqKgVsBr4CXnD3x8tV\nQxGRKpLuwMgjQDNgHHA1cA3wJ2B34LXo52Yz+01lVFJEpLKkOzDyS+Aydx9XYNlfzGwBcLO7dzKz\n+cAYwmM4RURqhHRbgu2B+cUs/4gwUAJhzmDziqiUiEhc0g2Cs4FhZrZb3oLo9e3A+9Gi3sDiiq2e\niEjlKsttc38FvjSz/xCePfwj4DOgn5mdBDwE/LxSaikiUknKMk+wFtADOBjYSugKv+3uKTPbA6Ay\n02tpikw8NEUmPpoiE58dniID4O45ZvYdsAZ4kfDMkV2ATXHnFhQRqShptQTNbE/gVaAjIfAZoft7\nEHCSu/+nMisJagnGRS3B+KglGJ+KSKo6gvBskWZse77I+cCHwPAdqZyISFVKNwgeD9zm7t/nLXD3\nb4HfAkdWRsVEROKQbhDMBeoVs7wlRZ48lwTvvvsuZ555JqeeeirXX399sd2a0sqMHz+e/v3787Of\n/YybbrqJzZs3x1n9GqVnz57MnTuXefPm8dxzz7Hbbrv9oMzAgQOZN28es2fP5plnnqFJkyaFPm/d\nujWLFy9WV387pk+fzqmnnsrJJ5/M1VdfXez3uqQy69evZ9CgQfTt25fevXszZsyYuKtfbmV5+PoI\nM/sJ4X7hhmZ2IjAKeKGyKlcdrVmzhltvvZUHHniAv/zlL7Ru3ZqHH3447TJvvfUWY8eOZfTo0Uyc\nOJFNmzbx3HPPVcWhVHvNmjVj1KhRnHPOOXTq1IklS5Zw++23FypzzDHHcO2119K7d2+6devG1KlT\nGTFiRP7n55xzDm+++SatWrWKu/o1ypo1a/jd737H8OHDmTJlCm3btuWBBx5Iu8zDDz9MixYtmDx5\nMhMmTGDs2LHMmzevuF1VO+kGwd8QEqvOBRoQHr/5V+Ct6LPEmD17NgceeCB77bUXAGeddRavvfYa\nBQeYSivz6quv8stf/pJGjRqRmZnJzTffTJ8+farkWKq7448/nn/+85/897//BWDMmDH079+/UJlO\nnToxbdo0vvzySwAmTZpEr169qF27Ni1atKBPnz7069cv9rrXNLNmzeKggw5i7733BuAXv/gFkydP\nLvS9Lq3M4MGD+c1vQihYuXIlW7ZsKbbVXh2lFQTdfbO7Xwc0IYwIdwKauvuvgfpl2aGZtTGzXmZW\ny8zalrnGVWz58uU0b77t7sDmzZuzYcMGsrOz0yrz2WefsWbNGi6//HLOPPNMHn/88RrzZYlbmzZt\nWLZsWf77L774gkaNGhU6X//4xz849thjads2fJXOP/98dt11V3bffXeWL1/OOeecw6JFi2Kve03z\n1Vdf0aJFi/z3LVq0+MH3urQyGRkZZGVlccMNN9C3b18OP/xw9tlnn1iPobzSfdpcjpnt4e7fu/tH\n7r7A3TeY2d6EUeO0mFlvQotyJLAn8LGZnVaOeleZkqYUZWZmplVm69atzJ49m/vuu4+xY8eybt06\nHnnkkUqpa01X0qyGnJyc/NezZs3i97//PePGjWPGjBnk5uayevVqXWcto9zc3GKXF/xep1Pmvvvu\nY/bs2axbt46RI0dWbCUrSYlB0MwuMLN3zexdwm1yf8l7X2D5m8CXZdjfEKAL8I27fwUcRbj/uMZo\n0aIFq1atyn//9ddf07BhQ+rVq5dWmT322IMePXrQoEEDateuTe/evVmwYEGsx1BTLFu2rFDLo1Wr\nVqxZs4bvvvsuf1mDBg2YOXMmRx55JEcffTSTJk0CwvUrSV+rVq1YuXLbPQ8rVqygUaNGhb7XpZWZ\nMWMGK1asAKB+/fr07t2bhQsXxncAO6C0luCLhGt+b0fvZ0av837eAh4DTizL/qLgB4C7z6f4ZK3V\nVrdu3fjwww/57LPPAJgwYQLHHXdc2mVOOOEE3nzzTTZu3EgqlWLatGl07NgxzkOoMd5++22OOOII\nfvSjHwFw6aWX8te//rVQmZYtWzJlypT8LvKNN97IhAkTYq9rTXfkkUeyYMEClixZAsC4cePo0aNH\n2mWmTJnCyJEjSaVSbN68mSlTptC1a9c4D6Hc0r1j5AJgnLtv2pGdRa3H84BX3P1QMzsauM/dt3u2\nqtMdIzNmzGD48OFs2bKFNm3acOedd7Js2TKGDh3KCy+8UGKZRo0akZOTw5gxY5g6dSo5OTnsv//+\n3HLLLTRo0KCKjyqobtNIevbsydChQ6lduzaffvopAwYMYO+99+bRRx+lW7duAPzqV7/isssuIzMz\nk9mzZ3PttdeycWPhmVvZ2dm0a9eO1atXV8VhFKu63THyzjvv8OCDD7Jlyxbatm3LPffcw9KlS7nl\nllt45ZVXSizTuHFjvv32W2677TYWLw6JpE444QSuuuqqQl3lqlQhzxgxs8MIt83lPXYzA9gV6OTu\nA9Lcxk+BpwjzCxcC+wFnuPs721u3OgXBnVl1C4I7s+oWBHdmFfGMkaHALcByQuLUL6J/s4CX0q2I\nu79nZl2BboRgOsfdV21nNRGRSpNuW3UAMNDdWwFLgeMIQfBNyjY6PAc4HXjH3V9VABSRqpZuENwd\nmBK9ngf81N2/AQYDZ5Vhf0OAnsASMxsVdbFFRKpMukFwGbBv9Ppj4NDo9XpCZpm0uPtUd/8F0AFY\nAIw2s5pxb42I7JTSTao6GhhvZhcBrwBvm9kKQnaZ4h7AVCIzyyJkqO5JmDA9vizri4hUpHRvm7sH\nuBbIdvf3gf8DziRkl7kk3Z2Z2QjC5OqBhKQM+0a344mIVIm00+sTWoBNANz9CTP7HJjt7uvLsI1v\ngS7u/mkZ1hERqTTp3jvclfBkuSsLLH4IcDM7JI3186ae/xPoZGb9Cv6UtdIiIhUl3ZbgH4AngJvy\nFrh7RzO7h5B6/+jtrH828DfgqmI+SwET06yHiEiFSve2uWzgIHf/X5HlPwI+dPcypdOK1s0Astx9\nSzrldcdIPHTHSHx0x0h8KuJBS59SfKKE44CvilleLDM7ysxuNrNdCF3jdWbWf3vriYhUlnS7w8OA\nP5rZkYTgBSGxan/gsjLs7z7C7Xc/I9yC14+Qnl/TZESkSqQ7RWYccApQh3AL3QVAXeB4dy/LAzJq\nuftbhFblK+6+hG0JGUREYpf2FBl3z8sjuCNqmdkRQG/gTjM7EKi9g9sUESm3uJN93UmYJP1k1Aqc\nDNwccx1ERPKlnU+wMphZLXfP2X5JjQ7HRaPD8dHocHx2OJ9gRTGzA4AbgKaEpKyYGe5+apz1EBHJ\nU6YgaGaZwF6EnIKZ7l7WR3r9kfC0uXeoYc8WEZGdU7qZpbOAuwh3fGQRUmHdbWZbgQHunl3a+gXs\n6u6DylVTEZFKkO7AyB2E1FcnAXlPsBlOmCv4QBn2t9jMWpahvIhIpUq3O3w2cK67zzKzFIC7zzSz\ni4FJhNRY6cgEPjKzfwLf5y3UNUERqSrpBsFmwNfFLM8mTJpO18vRj4hItZBuAoVXgFWEu0W+BQ4G\n1hDm/G1199PS3aGZtYnWnwq0cvel6aynKTLx0BSZ+GiKTHwqIoHCFYTA9TWh5fdXwnNH2gJpD3SY\nWS/C6PBIQmr9j80s7QAqIlLR0uoOu/sXwBFmdjzw42g9B95w99wy7O82oAvwmrt/ZWZHAc8SriuK\niMQu3czSmdEcwWnAY4REqm/kfVaW/bl7fuotd5+P5guKSBVKd2BkK6UHq3QzwXxnZu3ytmVmR7Nt\nyo2ISOzSDYLdi1nvR4Snzg0uw/5+S2hBtjSz2cB+wBllWF9EpEKle03wnWIWv21mDjzMdp4RYman\nu/vLhISsXYFuhNbjHHdfVbYqi4hUnB1NoLAKsDTK3U6YHzjb3Q8FXt/B/YqIVIh07x2+uJjFuwEX\nArPT2MS3ZvYJ0MbMPiz6obsfnE49REQqWrotwVuKvE8Bm4G/k15S1FOAQ4AnKf6xmyIiVSLdIDgQ\nmOXu5Z3i/qa7dzGzf5VwfVFEpEqkGwSfB44BPirnfpqb2e+Aw83s2qIfuvuD5dyuiMgOSTcILmDH\nguAA4BeEW+4OLLA8A02WFpEqlG4ChUlAH2A1sIQiE5zd/Zg0ttGacKvd4mjRLOAeJVCoXpRAIT5K\noBCfinjGyAfRT7mYWVtC4oQngLeAXQkTsOeaWVd3/7y82xYR2REltgTN7JfAeHfftKM7MbNngbeK\nPqjdzC4Cerj7+dvbhlqC8VBLMD5qCcanvKm0ngYaVVAdDi0aAAHc/WngiArah4hImZUWBEuMnOVQ\n2rZ2uKUpIlJe20uDVVFd0K1m1qrowmiZgqCIVJntDYwsN9v+rcHuvr1UWo8DT5vZz939WwAz2xN4\nDng0nYqKiFSG7QXB/sDaHd2Juz9uZu2BL8xsIVCbkEZreHRdcLvq1Kmzo9WQNGRnp/sIadlRpVyr\nlwpW2lTA0kaHc4CW7l7cU+bKJZor2CV6O8fdvyzD6hodlp2KgmB8UqlUueYJVvj/UPSsklJzD4qI\nxKm0gZFnKfCAdBGRnVFat81VEzWmoiLpUHc4PqV1h8vypDgRkZ2OgqCIJJqCoIgkmoKgiCSagqCI\nJJqCoIgkmoKgiCSagqCIJJqCoIgkmoKgiCSagqCIJJqCoIgkmoKgiCSagqCIJJqCoIgkmoKgiCSa\ngqCIJJqCoIgkmoKgiCSagqCIJJqCoIgkmoKgiCSagqCIJJqCoIgkmoKgiCSagqCIJJqCoIgkmoKg\niCSagqCIJJqCoIgkmoKgiCSagqCIJJqCoIgkmoKgiCSagqCIJJqCYDlMnz6dvn370rNnTwYNGsSG\nDRvSLrN+/XoGDRpEnz596NWrF6NHj467+jWKznV8evXqxYIFC1i0aBEvvPACu+222w/KXHnllSxa\ntIh58+bx/PPP06RJkx+UeemllxgxYkQcVa4YqVSqpvxUC6tXr0517do19emnn6ZSqVTq3nvvTQ0Z\nMiTtMnfccUdq2LBhqVQqlcrOzk5179499cEHH8RU+5plZz/XQLX5adasWWrFihWp9u3bp4DU3Xff\nnRo5cmShMscdd1xq6dKlqdatW6eA1HnnnZeaMGFCoTI33HBD6uuvv06NGDGiyo+p4E+qlNgSayDr\n0KHDH3dg/Wph0qRJqQEDBuS/X7p0aerQQw9N5ebmplUmNzc3tWXLllQqlUotWbIkdeSRR6YWL14c\n3wHUIDv7ua7qwFDw55xzzkm9+uqr+e/32muv1DfffFOozHXXXZd6+umn89/Xq1cvtXHjxlTt2rXz\ng+Tbb7+duv3222tUEIy7O/wTM8uIeZ8Vavny5bRo0SL/fYsWLdiwYQPZ2dlplcnIyCArK4vrr7+e\nPn36cMQRR7DPPvvEegw1hc51fNq2bcvSpUvz3y9btoxGjRoV6hK///779OjRg3bt2gFw0UUXseuu\nu7L77rvTsmVLHn74Yc4991xycnJir/+OiDsIfgV8ZGZPmdnwvJ+Y67BDcnNzi12emZlZpjL3338/\nc+bMYd26dYwcObJiK7mT0LmOT8HzVVDBgDZjxgyGDh3Kyy+/zN///ndyc3NZvXo1ubm5jBs3jmuu\nuYbly5fHVeUKE3cQnA2MBz4DVhf4qTFatmzJypUr89+vWLGCRo0aUa9evbTKzJgxgxUrVgBQv359\nevfuzcKFC+M7gBpE5zo+n3/+OS1btsx/37p1a9asWcN3332Xv6xBgwa88847dO7cmcMPP5yXXnoJ\ngH333Zd99tmHBx98kHnz5jFw4ED69+/PmDFjYj+O8og1CLr7UOBeYCJwB3BvtKzGOOqoo1iwYAFL\nliwBYNy4cRx//PFpl3n99dcZOXIkqVSKzZs38/rrr9O1a9c4D6HG0LmOzxtvvEHXrl1p3749AAMH\nDmTSpEmFyrRq1Yrp06fnd5FvueUWxo4dy5w5c2jXrh2dOnWiU6dOPP7444wfP54BAwbEfhzlUtoF\nw4r+6dChQ5cOHTp82aFDh887dOjQpkOHDqs7dOjw0zTXrzamT5+e6tu3b+rkk09OXXbZZam1a9em\nPvzww9Spp55aaplUKpVat25d6pprrkn17t071adPn9RDDz2UysnJqapDqfZ25nNNNRgwKPhzyimn\npObPn59auHBhavLkyakmTZqkOnfunJo3b15+mSuuuCK1cOHC1KJFi1JPPPFEqk6dOj/YzpAhQ2rU\nwEhG+L+Ih5nNAH4F/NndO5lZL2Coux+exurxVVQkBhkZNXqMsEZJpVIlnuy4rwnWc/f8izLu/hqQ\nFXMdRETyxR0Et5hZE6JWnZlZzPsXESkk7lbYMOAdoIWZjQVOAi6LuQ4iIvlivSYIYGbtgROBWsDf\nCnaPt0PXBGWnomuC8alO1wQBvgfmAO8Bdczs0Cqog4gIEHN32MyGAdcBBaeVp4B946yHiEieuK8J\nngfs7e4rYt6viEix4u4Or1QAFJHqJJaBkQLX/QYSrgk+D2zJ+9zdP0hjMxoYkZ2KBkbiU9rASFzd\n4ZeKvD+1wGtdExSRKhNLEHT3fQDMrI27Lyv4mZl1jKMOIiLFias73DR6OQ04DsggtAB3AWa6+35p\nbEbdYdmpqDscn+rQHR5LmCANhfMH5hDSaomIVIm4s8g85e4Xl3N1tQRlp6KWYHxKawnG1R3+sbsv\nKunuEI0OSxIpCManOnSH7wf68MNRYtDosIhUodgTKOyAGlNRkXSoJRif6tASBMDMGgB3A/sDPwfu\nAq5z9w3iPtd8AAAMuElEQVRx1kNEJE/ct80NB9YBzYGNQENgdMx1EBHJF3cQ7OTug4Et7v4dcC5w\nSMx1EBHJF3cQLPpo+lpA8U/PFhGJQdxB8F0zuweoa2Y9gZeB6THXQUQkX9xB8EZgA+G64DBgPiHJ\nqohIlYhrsvTTFJ7ikjdcnQJS7n5JGpvRFBnZqWiKTHyqwxSZfxezrBlwDbAkpjqIiPxAlUyWNrMT\ngGeB14FB0Ujx9qglKDsVtQTjUx1aggCYWRZhgvSFwOXu/mKc+xcRKSq2IBg9b3gckA0c6u5L49q3\niEhJYhkdNrOLgfeBl939WAVAEaku4hodziVMiv6eH44Sp9y9YRqb0TVB2anommB8qsM1wX1i2o+I\nSJkolZZIFVFLMD6ltQTjvmNERKRaURAUkURTEBSRRFMQFJFEUxAUkURTEBSRRFMQFJFEUxAUkURT\nEBSRRKtJd4yIiFQ4tQRFJNEUBEUk0RQERSTRFARFJNEUBEUk0RQERSTRYn3a3M7AzFLAHu6+qsCy\nC4Ez3b3Pdta9FVjg7pMqt5bVk5kNBC4HahOS5H4ADHb3z83sJGAM8DVwAvAa0Bi41d1fqsA6PAGM\nc/e3KmqbNYGZ7Q18Cgxw9ycKLL8eONDdLyxl3deA6919oZm9AZzj7qvMbAnhe/+Pyqx7ZVMQjFcP\nYGFVV6IqmNn9wE+APu6+1MwygfOA2WbWBfgFMMbdh5nZMUBzd29f0fVw90sreps1SC5wv5m96+6f\npLuSu/cq8PbEiq9W1VIQrGBm1gEYCTQAWgHzgf7AJcBhwH1mlgP8FbgHOBaoBcwjPIj+26qod2Uy\nszbAQKCtu68FcPdc4I9m1hlwYBPwffT+IKC1mc0HugGdCOeqPuEX+TZ3fzVqgZ8eLdsP2Az80t3/\nbWb9gJujz3KAG9z9XTObDjwCHAo0dPcrozqeDAx19y5m9tPi9lepJyke3wMPAGPNrJu7b877wMx2\noYTvY16LD7giKj7NzPIC46/M7HFgT+A5dx8cba8v4fzvAnxHaEnONrPbCP+nLYEP3f28SjzetOia\nYPlMM7P5eT/A7QU+GwA86+7dgPaEh0z1dveRwD8Iv4wvA78FtgKd3f0nwJfA3bEeRXy6AB/nBcAi\n3gL+A/wFeMjdTwcuBf7r7ocAdYCngfPd/VDgVOAxM2sXrX8scJW7HwjMAm6Ilt8H/NrdDwNuAY4r\nst8ngP7RLz/ARcAYM2uynf3VdHcSnv39+yLLt/t9dPeLopfdCzw2d2N0jo8ArjOztma2X7T9Xu7e\nCbgMmGhm9aN19iI8e7zKAyCoJVhe3Yu7Jhi9vRE40cx+A3QgtAYbFLONPoRrXieaGYS/mF9XYp2r\nWu0Slu9K6Q/Ryms1vBKdJ6LyB0ev/+nuy6LXHwD9otfjgJfN7K/Am8C9BTfq7v8zswXAqWb2NnA8\nobV+TCn7+3w7x1jtuXuumZ0HzDOzqQU+Ku/38flou8vNbAWhRdiFcA7fLnAOcwmNAoA57r51R4+l\noigIVryxhPP6AqHL247wfOWiagFXu/vrAGbWgNDq2RnNAfYzsxbuvrzIZ92B94B6Jaxbi9CK7JK3\nwMxaASuBcwldvDwponPt7oPN7EngJOBC4LdRV7ugJ4BfAs2Bl919g5mVtr+dQjQQNRB4FvhjtLi8\n38ctBV7nnf9awNvu3j/vAzNrS2hdng5s2OGDqEDqDle8nsDt7j6e8KXoQvhSQOhu5LWIpgJXmtku\n0SDBGOCuuCsbB3f/AhhOuBbVOm+5mV0EnEG4FlWSvAB6TLTOIcBiQgu7WGaWFV3Hqu/ujwO/Bvbn\nh63Rl4HOhEsYY8q7v5rI3ScArwPXRIvS/T7mUHKrPs/fgJPM7McA0fXDD6mmf+QVBCve7wjdsH8A\njwPvsK0bMJkwOncBcAewhHABeiHhL+h1sdc2Ju5+E/AnYJKZ/dvMFhOmwnRz989KWW8lIVDeF3Vf\nnyNcryttna2EX+7nzewDYAJwsbtvKlJuEzAeyHT398u7vxpsEJB3XOl+HycCM83swJI26u4fEa4D\njovO4R3Aqe6eXXFVrzhKpSUiiaaWoIgkmoKgiCSagqCIJJqCoIgkmoKgiCSaJkvvRKK5cXsVWLQV\nWAqMdvcKuyXPzGYCb7n7bWb2DJC1vVugzCwD+FVUl9xy7PME4E13L27iebntaL0qYP/PkMb5k8qj\nluDO5zrCLUstgX2BocAwM/tlJe3varbdWF+aY4DHqH7fuaquV7rnTyqJWoI7n2+L3Jr2rJmdTbin\n9o8lrFNu7r4uzaIV2oKrQFVarzKcP6kkCoLJsJWQZoooldS/gZOBuoQcf3UI6aVOBFYTboq/NS/V\nkpmdTri1rTXhftv8VlPR7pyZ/YKQtWWfaD9XA18B06JVtphZd3efbmaXEbKX7ElIOfZ/7v73aDsN\ngVGEG/u/BJ4s7QDNrBPwB+BwYDkwzN2fij7rQ8j0cwAhZdcUwq1yTctRr7rACODnhHtgb43q2d7d\nl0RZaO4BTovO72RClps1ZnYc4a6ZV4DzgYeAvYucv58RMr3sAywCfufuU6LPDiKkaesMrAf+DNxY\nnZIR1ETVrWsiFcjMakd59U4CCmazvoiQVOA0QtB7GVhL+OU6lxB47oq2cQAhGcRj0ed1CJlditvf\n8YTbzEYSsq5MIySRWEu4FQ2gDfBelG/uDuD/CPkCXwf+ZmYto3KPAz8mpMoaBFxbynE2A94GPo62\nNZiQ/uooM9sHeKnA9n5OSG47kHC9tKz1Gg4cRfgj0h/4DdvuDSc6l4cAfQmZaSw6J3laAw0J+Qyf\nKXIcP4nK3k3IqTiacAvmIVGRPxEC40HAWYRAeklJ50XSo5bgzucRM/tD9LouIaHlQ+7+5wJlXnf3\nmZAfuPYFurp7DrDIzK4A3jCzGwkBc5a7PxSVv5KQY684A4Hx7v5oVPYmQnezEbAmKrPC3bdGqcbu\nLvCogTujwY9LzWw44Zf8BHf/INrWMEILrDj9CS2jK6JjcDPbnRCcsgjZUUZHZZeY2VtAR3fPMbOy\n1OshQtaZvu4+O6rXIELLEjM7mBC0D3D3j6Nl5wEfm1nHAvW9193/G31e8DiuB55y97yg+V8LWbev\nIgS7vQl/VD6LUoGdQvgjJjtAQXDnM5SQMABgI/BVFBgKWlLg9f6EPHLrCvxCZhDyye1F6EIuyPvA\n3bdEN8UX5wBCdzmvbC6hpYSZ7Vuk7P7A783sjgLLdgWWEfIw1iq4X0JC2pIcAMwveJzu/kjeazPb\nZGaDgQOBjtHP2BK2VVq9fkw4L38v8NnsIuuuzwuAUT0Wmdna6LO8HJRLStn3QWZWsHVXG3g/ej2M\n0Eq8zMxeJzwr5Z8lbEvSpCC481np7v/ZTpmNBV5nEVJFFfeQqLzswUUHD7YULRjZXMLy4mQRRrLf\nKLJ8A9C2mP2WtM9S9xt1MWcRrs3NAB5kW/qostarRTH1Kvi6YG7DgmpRuMu8sYRyWcD9hMzWBW0C\ncPf7zGw84TJGb0JGnmHuflsJ25M0KAiKE4LO6rz092Z2FGFA43zC4MYxeYWjpKMHA8W1QBYTrqPl\nlc0APiJczysaIJzwzJH/FCj/GCH12GuEoHc4Ic8dBbdbwn5PN7PMvLl+ZvYUofVWj9CdP7vAfvaL\n1oEfZrUurV6vEgJuZ0K2aqLXBdfdzcz2L9AdPoBwDdAJLe7SOLBvkX0PBVab2WjCgMv97j4CGGFm\nNxOu4d62ne1KKRQE5Q3Coxj/HF3Dq0fo0i5w940WHlF5tYXHhY4nJChtU8K2HiYMIrwLTGfbCOxs\ntuVUPNTMPiS0yJ4ys0XATMKT5y4GRkUP93kOeNjCowvqEkacS/JnwujvQ2Y2kvBAq3MIAyDHAgdG\n19bWEK5bHs62VPl5WY7TqdcGM3s62k/eU+uGR/+m3N3N7FXCtKS8uX+PAjPdfX40Olyahwi5+t4n\ntFxPIAzynBr9XxwF7BX9P2UBp1D8HyMpA40OJ1x0Ha0vIWPwe2zrNl4afb44+vwswnSRZkQDAcVs\naxYhmeZNwL8Io6i9o7lw/yK06mYQHsAznjANZQihtXg68DN3nx9t7kpCEHqD0D0saVAkb65db0IW\n7w8JLaOL3f09QpCaRWi5vUcYXBjKtpZlWet1fXQe3iKMOucNOOV1yS8gtDLfjur+ESUPJBU9jjmE\nlt2AaL3/Ay5y99eiIv0Jo/NzonPzKWHQRHaAkqqKlEE0j+8td98QvT+cEGTru3tp1y2lmlIQFCkD\nM5tHeKrdXcBuhC7sSnf/eZVWTMpN3WGRsjmX0KWeR+gS/4/o0oHUTGoJikiiqSUoIommICgiiaYg\nKCKJpiAoIommICgiiaYgKCKJ9v8ERlLwLK9PcQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x111575dd8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix = confusion_matrix(y_test,y_preds)\n",
    "matrix_proportions = np.zeros((3,3))\n",
    "for i in range(0,3):\n",
    "    matrix_proportions[i,:] = confusion_matrix[i,:]/float(confusion_matrix[i,:].sum())\n",
    "names=['Hate','Offensive','Neither']\n",
    "confusion_df = pd.DataFrame(matrix_proportions, index=names,columns=names)\n",
    "plt.figure(figsize=(5,5))\n",
    "seaborn.heatmap(confusion_df,annot=True,annot_kws={\"size\": 12},cmap='gist_gray_r',cbar=False, square=True,fmt='.2f')\n",
    "plt.ylabel(r'True categories',fontsize=14)\n",
    "plt.xlabel(r'Predicted categories',fontsize=14)\n",
    "plt.tick_params(labelsize=12)\n",
    "\n",
    "#Uncomment line below if you want to save the output\n",
    "#plt.savefig('confusion.pdf')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1116a7c50>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD5CAYAAAAuneICAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFltJREFUeJzt3X+MVeWdx/H3MKOMtHfomAx1mzQhuu0nplmtwQWs0E6U\nXQKtsWvXrEtsad1aZd1qfyRqBde6oemqlS6QFVeQiEW3plrTLl0EU12FadEUa4Jd81VQ02ZTm1kz\nzEx3AgjO/nHOtDfT4f7kzpnyfF7JJOc89znnfs/D4XzuOffce9tGR0cxM7P0TCu6ADMzK4YDwMws\nUQ4AM7NEOQDMzBLlADAzS5QDwMwsUR2VHpR0CrAZmA1MB1YD/w08AIwCLwHXRcQ7kq4GrgGOAqsj\nYpuk04CtwCxgGFgeEf2S5gNr8747I+L2FmybmZlV0FbpcwCSPgecGxFfknQ68GL+tyYi/kvSvcAO\n4KfAk8D5QCewO5++DuiKiK9LugK4ICJukPQi8CngNeBHwMqI+HmlQvv7h5v6wEJ39wwGBkaaWUVL\nuK76uK76uK76nIx19fSU2o73WLVLQN8Dbs2n28hesc8BnsnbtgOLgLlAX0QcjohBYD9wDrAAeKK8\nr6QuYHpEHIiIUbIAWVT3VtWpo6O91U/RENdVH9dVH9dVn9TqqngJKCJ+CyCpBDwKrAK+lR+4Ibus\nMxPoAgbLFp2ovbxtaFzfM6sV2t09o+lB6OkpNbV8q7iu+riu+riu+qRUV8UAAJD0fuBx4J6IeFjS\nnWUPl4CDZAf0UpX2an0rava0rKenRH//cFPraAXXVR/XVR/XVZ+Tsa5KwVHxEpCk9wI7gZsiYnPe\n/HNJvfn0EmAX8DywUFKnpJnA2WRvEPcBS8v7RsQQcETSWZLagMX5OszMbBJVOwO4BegGbpU09l7A\nDcA6SacCLwOPRsQxSevIDuTTyN7UPSRpA7BF0m7gCLAsX8e1wENAO9ldQM+d0K0yM7OqKt4FNJU0\nexfQyXhq10quqz6uqz6uqz5NXgJq+C4gMzM7STkAzMwS5QAwM0uUA8DMLFFVPwdgZtVd8tUfFPK8\nm2++qJDntZODzwDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLl\nADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS1RNXwctaR5wR0T0SvoucEb+0GxgT0RcIWktsAAY++HK\nS8l+CH4rMCtvXx4R/ZLmA2uBo2Q/Cn/7idogMzOrTdUzAEk3ApuAToCIuCIieoG/Ag4CX867zgEW\nR0Rv/jcIrAD2RcRC4EFgVd73XmAZWWDMk3TeidskMzOrRS2XgA4Al03QfjuwPiJ+LWka8AHgPkl9\nkq7K+ywAnsintwOLJHUB0yPiQESMAjuARU1thZmZ1a3qJaCIeEzS7PI2SbOAi/n9q/93AeuBNUA7\n8LSknwFdwGDeZxiYmbcNla1uGDizWh3d3TPo6Giv1q2inp5SU8u3iuuqz1Stqwi1jMVUHS/XVZ9W\n1NXoT0L+NfBwRBzL50eAtRExAiDpKeBcsgP9WNUlsktG5W3l7RUNDIw0WGqmp6dEf/9w9Y6TzHXV\nZ6rWVZRqYzFVx8t11aeZuioFR6N3AS0iu6Qz5oNAn6R2SaeQXfp5AegDluZ9lgC7ImIIOCLpLElt\nwGJgV4N1mJlZgxo9AxDw2thMRLws6TvAHuBt4MGI+IWk14EtknaT3RG0LF/kWuAhsstFOyPiuUY3\nwMzMGlNTAETEG8D8svkPTdDnLuCucW0jwOUT9N1Tvj4zM5t8/iCYmVmiHABmZolyAJiZJcoBYGaW\nKAeAmVmiHABmZolyAJiZJcoBYGaWKAeAmVmiHABmZolyAJiZJcoBYGaWKAeAmVmiHABmZolyAJiZ\nJcoBYGaWKAeAmVmiHABmZolyAJiZJaqm3wSWNA+4IyJ6JZ0HbANezR/eEBGPSLoauAY4CqyOiG2S\nTgO2ArOAYWB5RPRLmg+szfvujIjbT+xmmZlZNVXPACTdCGwCOvOmOcCaiOjN/x6RdAZwPXAhsBj4\npqTpwApgX0QsBB4EVuXruBdYBiwA5uWhYmZmk6iWS0AHgMvK5ucAH5f0rKT7JZWAuUBfRByOiEFg\nP3AO2QH+iXy57cAiSV3A9Ig4EBGjwA5g0QnaHjMzq1HVS0AR8Zik2WVNzwObImKvpJXAbcCLwGBZ\nn2FgJtBV1l7eNjSu75nV6ujunkFHR3u1bhX19JSaWr5VXFd9pmpdRahlLKbqeLmu+rSirpreAxjn\n8Yg4ODYNrAeeBcqrKwEHyQ70pQpt5e0VDQyMNFDq7/X0lOjvH25qHa3guuozVesqSrWxmKrj5brq\n00xdlYKjkbuAdkiam09fDOwlOytYKKlT0kzgbOAloA9YmvddAuyKiCHgiKSzJLWRvWewq4E6zMys\nCY2cAawA1kt6G3gT+EJEDElaR3YgnwasjIhDkjYAWyTtBo6QvfELcC3wENBOdhfQc81uiJmZ1aem\nAIiIN4D5+fQLZHf7jO+zEdg4rm0EuHyCvnvG1mdmZsXwB8HMzBLlADAzS5QDwMwsUQ4AM7NEOQDM\nzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4A\nM7NEOQDMzBLlADAzS5QDwMwsUTX9JrCkecAdEdEr6cPAeuAYcBj4TET8RtJaYAEwnC92KdkPwW8F\nZuXtyyOiX9J8YC1wlOxH4W8/kRtlZmbVVT0DkHQjsAnozJvWAl+MiF7g+8BNefscYHFE9OZ/g8AK\nYF9ELAQeBFblfe8FlpEFxjxJ552g7TEzsxrVcgnoAHBZ2fwVEfFiPt0BHJI0DfgAcJ+kPklX5Y8v\nAJ7Ip7cDiyR1AdMj4kBEjAI7gEXNboiZmdWn6iWgiHhM0uyy+V8DSPoI8A/AR4F3kV0WWgO0A09L\n+hnQBQzmiw4DM/O2obKnGAbOrFZHd/cMOjraq29RBT09paaWbxXXVZ+pWlcRahmLqTperqs+rair\npvcAxpP0N8BK4OP5Nf12YG1EjOSPPwWcS3agH6u6BBwc11beXtHAwEgjpf5OT0+J/v7h6h0nmeuq\nz1StqyjVxmKqjpfrqk8zdVUKjrrvApJ0Jdkr/96IeC1v/iDQJ6ld0ilkl35eAPqApXmfJcCuiBgC\njkg6S1IbsBjYVW8dZmbWnLrOAPJX+uuAXwLflwTwTETcJuk7wB7gbeDBiPiFpNeBLZJ2k90RtCxf\n1bXAQ2SXi3ZGxHMnZGvMzKxmNQVARLwBzM9nTz9On7uAu8a1jQCXT9B3T9n6zMysAP4gmJlZohwA\nZmaJcgCYmSXKAWBmligHgJlZohwAZmaJcgCYmSXKAWBmligHgJlZohwAZmaJcgCYmSXKAWBmligH\ngJlZohwAZmaJcgCYmSXKAWBmligHgJlZohwAZmaJcgCYmSWqpt8EljQPuCMieiX9KfAAMAq8BFwX\nEe9Iuhq4BjgKrI6IbZJOA7YCs4BhYHlE9EuaD6zN++6MiNtP9IaZmVllVc8AJN0IbAI686Y1wKqI\nWAi0AZdKOgO4HrgQWAx8U9J0YAWwL+/7ILAqX8e9wDJgATBP0nknbpPMzKwWtVwCOgBcVjY/B3gm\nn94OLALmAn0RcTgiBoH9wDlkB/gnyvtK6gKmR8SBiBgFduTrMDOzSVT1ElBEPCZpdllTW37ghuyy\nzkygCxgs6zNRe3nb0Li+Z1aro7t7Bh0d7dW6VdTTU2pq+VZxXfWZqnUVoZaxmKrj5brq04q6anoP\nYJx3yqZLwEGyA3qpSnu1vhUNDIw0UOrv9fSU6O8fbmodreC66jNV6ypKtbGYquPluurTTF2VgqOR\nu4B+Lqk3n14C7AKeBxZK6pQ0Ezib7A3iPmBped+IGAKOSDpLUhvZewa7GqjDzMya0MgZwFeBjZJO\nBV4GHo2IY5LWkR3IpwErI+KQpA3AFkm7gSNkb/wCXAs8BLST3QX0XLMbYmZm9akpACLiDWB+Pv0K\n8LEJ+mwENo5rGwEun6DvnrH1mZlZMfxBMDOzRDkAzMwS5QAwM0uUA8DMLFEOADOzRDkAzMwS5QAw\nM0uUA8DMLFEOADOzRDkAzMwS5QAwM0uUA8DMLFEOADOzRDkAzMwS5QAwM0uUA8DMLFEOADOzRDkA\nzMwS5QAwM0tUIz8Kj6TPAp/NZzuBDwMXANuAV/P2DRHxiKSrgWuAo8DqiNgm6TRgKzALGAaWR0R/\noxthZmb1aygAIuIB4AEASf8KbAbmAGsi4u6xfpLOAK4HzicLit2SngRWAPsi4uuSrgBWATc0vhlm\nZlavpi4BSTof+FBE3EcWAB+X9Kyk+yWVgLlAX0QcjohBYD9wDrAAeCJfzXZgUTN1mJlZ/Ro6Ayhz\nC3B7Pv08sCki9kpaCdwGvAgMlvUfBmYCXWXtY20VdXfPoKOjvalie3pKTS3fKq6rPlO1riLUMhZT\ndbxcV31aUVfDASDpPYAi4um86fGIODg2DawHngXKqy4BB4GhsvaxtooGBkYaLRXIBq+/f7ipdbSC\n66rPVK2rKNXGYqqOl+uqTzN1VQqOZi4BfRT4cdn8Dklz8+mLgb1kZwULJXVKmgmcDbwE9AFL875L\ngF1N1GFmZg1o5hKQgNfK5lcA6yW9DbwJfCEihiStIzvATwNWRsQhSRuALZJ2A0eAZU3UYWZmDWg4\nACLirnHzLwAXTtBvI7BxXNsIcHmjz21mZs3zB8HMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAz\nS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDM\nzBLlADAzS5QDwMwsUQ3/JrCkF4ChfPZ14BvAA8Ao8BJwXUS8I+lq4BrgKLA6IrZJOg3YCswChoHl\nEdHf8FaYmVndGjoDkNQJtEVEb/73OWANsCoiFgJtwKWSzgCuJ/ux+MXANyVNB1YA+/K+DwKrTsC2\nmJlZHRo9AzgXmCFpZ76OW4A5wDP549uBvwSOAX0RcRg4LGk/cA6wALizrO+tDdZhZmYNajQARoBv\nAZuAD5AdxNsiYjR/fBiYCXQBg2XLTdQ+1lZRd/cMOjraGyw309NTamr5VnFd9ZmqdRWhlrGYquPl\nuurTiroaDYBXgP35Af8VSW+RnQGMKQEHyd4jKFVpH2uraGBgpMFSMz09Jfr7h5taRyu4rvpM1bqK\nUm0spup4ua76NFNXpeBo9C6gq4C7ASS9j+wV/U5JvfnjS4BdwPPAQkmdkmYCZ5O9QdwHLB3X18zM\nJlGjZwD3Aw9I2k12189VwP8CGyWdCrwMPBoRxyStIzvATwNWRsQhSRuALfnyR4BlzW6ImZnVp6EA\niIjjHbQ/NkHfjcDGcW0jwOWNPLeZmZ0Y/iCYmVmiHABmZolyAJiZJcoBYGaWqIa/C8jMLCVX/fNT\nhT33f9x9aUvW6zMAM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NE\nOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLV0NdBSzoF2AzMBqYDq4FfAduAV/NuGyLiEUlXA9cA\nR4HVEbFN0mnAVmAWMAwsj4j+ZjbEzMzq0+jvAVwJvBURn5Z0OvAi8E/Amoi4e6yTpDOA64HzgU5g\nt6QngRXAvoj4uqQrgFXADU1sh5mZ1anRAPge8Gg+3Ub26n4OIEmXkp0FfAmYC/RFxGHgsKT9wDnA\nAuDOfPntwK0N1mFmZg1qKAAi4rcAkkpkQbCK7FLQpojYK2klcBvZmcFg2aLDwEygq6x9rK2i7u4Z\ndHS0N1Lu7/T0lJpavlVcV32mal1FqGUspup4ua76tKKuhn8SUtL7gceBeyLiYUnviYiD+cOPA+uB\nZ4HyqkvAQWCorH2sraKBgZFGSwWywevvH25qHa3guuozVesqSrWxmKrj5brq12hdlYKjobuAJL0X\n2AncFBGb8+Ydkubm0xcDe4HngYWSOiXNBM4GXgL6gKV53yXArkbqMDOzxjV6BnAL0A3cKmns+v1X\ngG9Leht4E/hCRAxJWkd2gJ8GrIyIQ5I2AFsk7QaOAMua2gozM6tbo+8B3MDEd+1cOEHfjcDGcW0j\nwOWNPLeZmZ0Y/iCYmVmiHABmZolyAJiZJcoBYGaWKAeAmVmiHABmZolq+JPAf2wu+eoPCnnezTdf\nVMjzmplV4zMAM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDMzBLlADAzS5QDwMwsUQ4AM7NEOQDM\nzBLlADAzS5QDwMwsUYV9GZykacA9wLnAYeDzEbG/qHrMzFJT5BnAJ4HOiLgAuBm4u8BazMySU2QA\nLACeAIiIPcD5BdZiZpacttHR0UKeWNIm4LGI2J7P/xI4MyKOFlKQmVliijwDGAJKZfPTfPA3M5s8\nRQZAH7AUQNJ8YF+BtZiZJafIn4R8HPgLST8B2oDPFViLmVlyCnsPwMzMiuUPgpmZJcoBYGaWKAeA\nmVmiinwT+ISo9pUSki4B/hE4CmyOiI2T8TUUNdT1t8CX8rr2AX8fEe9IeoHsFlmA1yPihL45XkNd\nXwY+D/TnTdcAr1ZaptV1SToD+G5Z9w8DN0fEva0er7L65gF3RETvuPZC9q8a6ipk/6qhrkL2r0p1\nFbl/SToF2AzMBqYDqyPih2WPt3T/+qMPAMq+UiK/nfRu4FL43eB+G/hz4P+APkk/BC483jKTVNdp\nwGrgzyJiRNK/A5+QtBNoG/+fZrLqys0BPhMRe8caJF1WZZmW1hURbwK9eS0XAN8ANkrqpPXjhaQb\ngU+T7UPl7UXuX5XqKnL/Om5duaL2r+PWVfD+dSXwVkR8WtLpwIvAD/NaWr5/nQyXgCp9pcTZwP6I\nGIiII8Bu4KNVlpmMug4DH4mIkXy+AzhElugzJO2U9FT+jzuZdUH2H/RrknZL+lqNy0xGXUhqA9YD\nKyLiGJMzXgAHgMsmaC9y/6pUV5H7V6W6oLj9q1pdRe1f3wNuzafbyF7pj2n5/nUyBEAXMFg2f0xS\nx3EeGwZmVlmm5XVFxDsR8RsASV8E3g08CYwA3wIWA9cCD01mXbnv5s99EbBA0idqWGYy6gK4BPhF\nREQ+PxnjRUQ8Brw9wUNF7l/Hravg/avSeEFx+1e1uqCA/SsifhsRw5JKwKPAqrKHW75/nQyXgCp9\npcT4x0rAwSrLTEZdY9e87wQ+CHwqIkYlvUKW+KPAK5LeAv4E+NVk1JW/AvqXiBjM538EnFdtW1pd\nV5krgbVl85MxXpUUuX9VVOD+VammIvevWhSyf0l6P9kHY++JiIfLHmr5/nUynAFU+kqJl4EPSDpd\n0qlkp08/rbLMZNQF8G9AJ/DJslP1q8i/FlvS+8iS/teTWFcX8JKkd+f/WS8C9tawLa2ua8z5wE/K\n5idjvCopcv+qpqj9q5Ii969aTPr+Jem9wE7gpojYPO7hlu9fJ8MZwB98pYSkZcC7I+I+SV8BdpCF\n3eaI+B9Jk/E1FMetC/gZ8HfALuApSZC98rgfeEDSbmAUuKoFr4SqjdctwNNk15F/HBH/mb+aLGy8\n8rp6gKH81diYyRivPzBF9q/j1kWx+9dx6yp4/6pWV1H71y1AN3CrpLH3AjYC75qM/ctfBWFmlqiT\n4RKQmZk1wAFgZpYoB4CZWaIcAGZmiXIAmJklygFgZpYoB4CZWaL+H1ff6+tI9kgAAAAAAElFTkSu\nQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11167aa20>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#True distribution\n",
    "y.hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x114cd3dd8>"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD3CAYAAAAT+Z8iAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEoxJREFUeJzt3X+MZWV9x/H3sqOOq8M6JIPUxIQ0xW8IKUiWAsqiG7AS\naMkaWlJKQOxWoNQKqAkiuxRp1rSIYGFTsS5uEKiVCBLptghEKO6uEnQpCRjzhUWJphEz0tnd0QkL\nLNs/7pn2up2Ze++5c3/s0/crIbnnuefM+dzHh889nLlzXbJv3z4kSeU6aNABJEm9ZdFLUuEsekkq\nnEUvSYWz6CWpcCODDjCXycnp2h8FGh9fxtTUzGLGWRTm6oy5OmOuzpSaa2JibMlc48Vd0Y+MLB10\nhDmZqzPm6oy5OvP/LVdxRS9J+k0WvSQVzqKXpMJZ9JJUOItekgpn0UtS4Sx6SSqcRS9JhbPoJalw\nQ/kVCNKwOvMT3xzYuTddecrAzq0Dm1f0klS4tq7oI+IE4LrMXBURXwMOq546HHgsM8+JiJuAlcB0\n9dxq4GXgTuDQavyCzJxcxPySpBZaFn1EXAGcD/waIDPPqcbHgUeAj1W7rgBOy8xfNh37ceCpzPx0\nRJwDrAMuW9RXIElaUDu3bp4Dzppj/FpgQ2b+PCIOAo4AvhQR2yJiTbXPSuBb1eP7gfd1G1iS1JmW\nV/SZeU9EHN48FhGHAqfyv1fzbwI2ADcCS4FHIuIHwMHArmqfaWB5O6HGx5d19XWdExNjtY/tJXN1\nZlhzDUqr+RjW+TJXZ3qRq+6nbv4Y+Gpm7q22Z4CbMnMGICIeBo4BdgOzqceAne388C6/eJ/JyenW\nO/aZuTozrLkGaaH5GNb5Mldnus0135tE3U/dvI/GrZhZ7wC2RcTSiHgdjVs2TwDbgDOqfU4HttQ8\nnySpprpFH8CPZzcy80fAHcBjwKPA7Zn5Q+AW4KiI2ApcROO+viSpj9q6dZOZzwMnNm0fNcc+1wPX\n7zc2A5zdXURJUjf8gylJKpxFL0mFs+glqXAWvSQVzqKXpMJZ9JJUOItekgpn0UtS4Sx6SSqcRS9J\nhbPoJalwFr0kFc6il6TCWfSSVDiLXpIKZ9FLUuEsekkqnEUvSYWz6CWpcBa9JBXOopekwo20s1NE\nnABcl5mrIuJYYDPwbPX0LZl5V0RcCFwMvAqsz8zNEfFG4E7gUGAauCAzJxf9VUiS5tWy6CPiCuB8\n4NfV0Argxsy8oWmfw4BLgeOAUWBrRDwEXAI8lZmfjohzgHXAZYv7EiRJC2nniv454Czgjmp7BRAR\nsZrGVf3lwPHAtszcA+yJiB3A0cBK4LPVcfcDVy9idklSG1oWfWbeExGHNw09DtyamdsjYi1wDfAk\nsKtpn2lgOXBw0/jsWEvj48sYGVnazq5zmpgYq31sL5mrM8Oaa1Bazcewzpe5OtOLXG3do9/PvZm5\nc/YxsAH4DtCcbgzYCexuGp8da2lqaqZGrIaJiTEmJ6drH98r5urMsOYapIXmY1jny1yd6TbXfG8S\ndT5180BEHF89PhXYTuMq/+SIGI2I5cCRwNPANuCMat/TgS01zidJ6kKdK/pLgA0R8QrwAnBRZu6O\niJtpFPlBwNrMfCkibgG+EhFbgZeBcxcruCSpPW0VfWY+D5xYPX4COGmOfTYCG/cbmwHO7jqlJKk2\n/2BKkgpn0UtS4Sx6SSqcRS9JhbPoJalwFr0kFc6il6TCWfSSVDiLXpIKZ9FLUuEsekkqnEUvSYWz\n6CWpcBa9JBXOopekwln0klQ4i16SCmfRS1LhLHpJKpxFL0mFs+glqXAj7ewUEScA12Xmqoh4J7AB\n2AvsAT6Ymb+IiJuAlcB0ddhq4GXgTuDQavyCzJxc5NcgSVpAyyv6iLgCuBUYrYZuAj6amauAbwCf\nrMZXAKdl5qrqn13AJcBTmXkycDuwbpHzS5JaaOfWzXPAWU3b52Tmk9XjEeCliDgIOAL4UkRsi4g1\n1fMrgW9Vj+8H3rcImSVJHWh56yYz74mIw5u2fw4QEe8G/gp4D/AmGrdzbgSWAo9ExA+Ag4Fd1aHT\nwPJ2Qo2PL2NkZGn7r2I/ExNjtY/tJXN1ZlhzDUqr+RjW+TJXZ3qRq6179PuLiD8B1gJ/kJmTEbEU\nuCkzZ6rnHwaOAXYDs6nHgJ3t/PypqZk6sYDGJE1OTrfesc/M1ZlhzTVIC83HsM6XuTrTba753iQ6\nLvqIOA+4GFiVmf9VDb8DuCsijqVxO2gl8BUav4Q9A3gcOB3Y0nFySVJXOir66sr9ZuCnwDciAuDR\nzLwmIu4AHgNeAW7PzB9GxE+Ar0TEVhqfwDl3UdNLklpqq+gz83ngxGrzkHn2uR64fr+xGeDsLvJJ\nkrrkH0xJUuEsekkqnEUvSYWz6CWpcBa9JBXOopekwln0klQ4i16SCmfRS1LhLHpJKpxFL0mFs+gl\nqXAWvSQVzqKXpMJZ9JJUOItekgpn0UtS4Sx6SSqcRS9JhbPoJalwFr0kFW6knZ0i4gTgusxcFRG/\nA9wG7AOeBj6Sma9FxIXAxcCrwPrM3BwRbwTuBA4FpoELMnOyB69DkjSPllf0EXEFcCswWg3dCKzL\nzJOBJcDqiDgMuBQ4CTgN+NuIeANwCfBUte/twLrFfwmSpIW0c0X/HHAWcEe1vQJ4tHp8P/B+YC+w\nLTP3AHsiYgdwNLAS+GzTvle3E2p8fBkjI0vbegFzmZgYq31sL5mrM8Oaa1Bazcewzpe5OtOLXC2L\nPjPviYjDm4aWZOa+6vE0sBw4GNjVtM9c47NjLU1NzbSz25wmJsaYnJyufXyvmKszw5prkBaaj2Gd\nL3N1pttc871J1Pll7GtNj8eAncDu6vFC47NjkqQ+qlP0/xERq6rHpwNbgMeBkyNiNCKWA0fS+EXt\nNuCM/faVJPVRnaL/BHBtRHwPeD1wd2a+ANxMo8gfBtZm5kvALcBREbEVuAi4dnFiS5La1dbHKzPz\neeDE6vEzwHvn2GcjsHG/sRng7K5TSpJq8w+mJKlwFr0kFc6il6TCWfSSVDiLXpIKZ9FLUuEsekkq\nnEUvSYWz6CWpcBa9JBXOopekwln0klQ4i16SCmfRS1LhLHpJKpxFL0mFs+glqXAWvSQVzqKXpMJZ\n9JJUOItekgo3UuegiPgQ8KFqcxR4J/AuYDPwbDV+S2beFREXAhcDrwLrM3NzN4ElSZ2pVfSZeRtw\nG0BE/AOwCVgB3JiZN8zuFxGHAZcCx9F4Q9gaEQ9l5p7uYkuS2lWr6GdFxHHAUZn5kYi4pTEUq2lc\n1V8OHA9sq4p9T0TsAI4Gvr/Qzx0fX8bIyNLauSYmxmof20vm6syw5hqUVvMxrPNlrs70IldXRQ9c\nBVxbPX4cuDUzt0fEWuAa4ElgV9P+08DyVj90amqmdqCJiTEmJ6drH98r5urMsOYapIXmY1jny1yd\n6TbXfG8StX8ZGxFvASIzH6mG7s3M7bOPgWOB3UDzmceAnXXPKUnqXDefunkP8O2m7Qci4vjq8anA\ndhpX+SdHxGhELAeOBJ7u4pySpA51c+smgB83bV8CbIiIV4AXgIsyc3dE3AxsofGmsjYzX+rinJKk\nDtUu+sy8fr/tJ4CT5thvI7Cx7nkkSd3xD6YkqXAWvSQVzqKXpMJZ9JJUOItekgpn0UtS4Sx6SSqc\nRS9JhbPoJalwFr0kFc6il6TCWfSSVDiLXpIKZ9FLUuEsekkqnEUvSYWz6CWpcBa9JBXOopekwln0\nklS42v/n4BHxBLC72vwJ8BngNmAf8DTwkcx8LSIuBC4GXgXWZ+bmrhJLkjpSq+gjYhRYkpmrmsbu\nA9Zl5r9HxBeB1RHxPeBS4DhgFNgaEQ9l5p7uo0uS2lH3iv4YYFlEPFj9jKuAFcCj1fP3A+8H9gLb\nqmLfExE7gKOB73eVWpLUtrpFPwN8DrgVOIJGsS/JzH3V89PAcuBgYFfTcbPjCxofX8bIyNKa0WBi\nYqz2sb1krs4Ma65BaTUfwzpf5upML3LVLfpngB1VsT8TES/SuKKfNQbspHEPf2yO8QVNTc3UjNWY\npMnJ6drH94q5OjOsuQZpofkY1vkyV2e6zTXfm0TdT92sAW4AiIi30bhyfzAiVlXPnw5sAR4HTo6I\n0YhYDhxJ4xe1kqQ+qXtF/2XgtojYSuNTNmuAXwIbI+L1wI+AuzNzb0TcTKP0DwLWZuZLi5BbktSm\nWkWfmS8D587x1Hvn2HcjsLHOeSRpENb83cMDOe+/3LC6Jz/XP5iSpMJZ9JJUOItekgpn0UtS4Sx6\nSSqcRS9JhbPoJalwFr0kFc6il6TCWfSSVDiLXpIKZ9FLUuEsekkqnEUvSYWz6CWpcBa9JBXOopek\nwln0klQ4i16SCmfRS1LhLHpJKtxInYMi4nXAJuBw4A3AeuBnwGbg2Wq3WzLzroi4ELgYeBVYn5mb\nuw0tSWpfraIHzgNezMzzI+IQ4Engb4AbM/OG2Z0i4jDgUuA4YBTYGhEPZeaeLnNLktpUt+i/Dtxd\nPV5C42p9BRARsZrGVf3lwPHAtqrY90TEDuBo4PtdpZYkta1W0WfmrwAiYoxG4a+jcQvn1szcHhFr\ngWtoXOnvajp0Glje6uePjy9jZGRpnWgATEyM1T62l8zVmWHNNSit5mNY58tcnelFrrpX9ETE24F7\ngS9k5lcj4i2ZubN6+l5gA/AdoDn1GLCTFqamZurGYmJijMnJ6drH94q5OjOsuQZpofkY1vkyV+e6\nyTXfm0StT91ExFuBB4FPZuamaviBiDi+enwqsB14HDg5IkYjYjlwJPB0nXNKkuqpe0V/FTAOXB0R\nV1djHwc+HxGvAC8AF2Xm7oi4GdhC401lbWa+1G3ohZz5iW/28sfPa9OVpwzkvJLUSt179JcBl83x\n1Elz7LsR2FjnPJKk7vkHU5JUOItekgpn0UtS4Sx6SSqcRS9JhbPoJalwFr0kFc6il6TCWfSSVDiL\nXpIKZ9FLUuEsekkqnEUvSYWz6CWpcBa9JBXOopekwln0klQ4i16SCmfRS1LhLHpJKpxFL0mFG+n1\nCSLiIOALwDHAHuDDmbmj1+eVJDX044r+A8BoZr4LuBK4oQ/nlCRV+lH0K4FvAWTmY8BxfTinJKmy\nZN++fT09QUTcCtyTmfdX2z8FfjszX+3piSVJQH+u6HcDY83ntOQlqX/6UfTbgDMAIuJE4Kk+nFOS\nVOn5p26Ae4Hfj4jvAkuAP+vDOSVJlZ7fo5ckDZZ/MCVJhbPoJalwFr0kFa4fv4xdFK2+SiEizgT+\nGngV2JSZG/v19QttZPtT4PIq21PAX2bmaxHxBI2PnwL8JDMX9RfVbeT6GPBhYLIauhh4dqFjep0r\nIg4Dvta0+zuBKzPzi72er+r8JwDXZeaq/cYHtr5a5BrI2moz20DW10K5Brm+IuJ1wCbgcOANwPrM\nvK/p+Z6tsQOm6Gn6KoXqY5o3AKvhfybw88DvAb8GtkXEfcBJ8x3Tx2xvBNYDv5uZMxHxz8AfRsSD\nwJL9/+XoV67KCuCDmbl9diAizmpxTE9zZeYLwKoqy7uAzwAbI2KUHs9XRFwBnE9jDTWPD3R9LZBr\nkGtrwWyVQa2veXMNcn0B5wEvZub5EXEI8CRwX5Wlp2vsQLp1s9BXKRwJ7MjMqcx8GdgKvKfFMf3K\ntgd4d2bOVNsjwEs03qGXRcSDEfFw9T9iP3NB41/ET0XE1oj4VJvH9CMXEbEE2ABckpl76c98PQec\nNcf4oNfXfLkGubZaZYPBra9WuQa1vr4OXF09XkLjyn1WT9fYgVT0BwO7mrb3RsTIPM9NA8tbHNOX\nbJn5Wmb+AiAiPgq8GXgImAE+B5wG/AXwTz3I1ur1f6069ynAyoj4wzaO6UcugDOBH2ZmVts9n6/M\nvAd4pY28fV1f8+Ua8NpaMFtlUOurVS4YzPr6VWZOR8QYcDewrunpnq6xA+nWzUJfpbD/c2PAzhbH\n9Cvb7D3pzwLvAP4oM/dFxDM03sH3Ac9ExIvAbwE/60eu6orm7zNzV7X9r8CxrV5Lr3M1OQ+4qWm7\nH/M1n0Gvr3kNcG21yjXI9dWOgayviHg7jT8i/UJmfrXpqZ6usQPpin6hr1L4EXBERBwSEa+n8Z88\n32txTL+yAfwjMAp8oOk/s9dQfWVzRLyNxjv3z/uY62Dg6Yh4c/Uv5SnA9jZeS69zzToO+G7Tdj/m\naz6DXl8LGdTaamWQ66sdfV9fEfFW4EHgk5m5ab+ne7rGDqQr+v/zVQoRcS7w5sz8UkR8HHiAxpvX\npsz8z4jo19cvzJsN+AHw58AW4OGIgMaVxJeB2yJiK7APWNODK5tWc3YV8AiNe73fzsx/q64Qez1n\nrXJNALurq6tZ/Ziv3zBE62vOXAx2bS2YbcDrq1WuQa2vq4Bx4OqImL1XvxF4U6/XmF+BIEmFO5Bu\n3UiSarDoJalwFr0kFc6il6TCWfSSVDiLXpIKZ9FLUuH+G1VPI8raqfeBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x111733160>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(y_preds).hist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
